As I stated in comment i would use a box layout for this.
JPanel panel = new JPanel();
panel.setLayout(new BoxLayout());
JButton button = new JButton("Button1");
button.setAlignmentX(Component.CENTER_ALIGNMENT);
panel.add(button);
button = new JButton("Button2");
button.setAlignmentX(Component.CENTER_ALIGNMENT);
panel.add(button);
button = new JButton("Button3");
button.setAlignmentX(Component.CENTER_ALIGNMENT);
panel.add(button);
add(panel);