0%

ubuntu server多节点部署havana的OVS-GRE模式

Posted on Edited on In

控制节点：

  1. 准备工作

    1. 配置网络，网卡配置说明，第一个网卡是用来连接外网，并且对外提供API接口，地址是在192.168.1.xx网段，注意要配置网关。
      第二个网卡是负责openstack内部管理连接，负责controller和compute之间管理端的数据连接，使用192.168.100.xx网段，内部网络不用配置网关。
      第三个网卡是负责openstack数据层面的连接，使用192.168.200.xx网段，内部网络不用配置网关。10是controller，11是compute1，12是compute2，如此类推。注意区分eth是千兆口还是万兆口。 vi /etc/network/interfaces
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      #For Exposing OpenStack API over the internet
      auto eth0
      iface eth0 inet static
      address 192.168.1.10
      netmask 255.255.255.0
      gateway 192.168.1.1

      #Not internet connected(used for OpenStack management)
      auto eth1
      iface eth1 inet static
      	address 192.168.100.10
      	netmask 255.255.255.0

      #DATA_INTERFACE
      auto eth3
      iface eth3 inet static
      	address 192.168.200.10
      	netmask 255.255.255.0
      service networking restart
    2. 配置本机的hostname hostname controller，更改hosts文件
      vi /etc/hosts
      1
      2
      3
      4
      5
      6
      127.0.0.1 localhost
      192.168.100.10 controller
      192.168.100.11 compute1
      192.168.100.12 compute2
      192.168.100.13 compute3
      192.168.1.12 controller_ext
    3. 更新教育网的源
      vi /etc/apt/sources.list
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      deb http://mirror.bjtu.edu.cn/ubuntu/ precise main multiverse restricted universe
      deb http://mirror.bjtu.edu.cn/ubuntu/ precise-backports main multiverse restricted universe
      deb http://mirror.bjtu.edu.cn/ubuntu/ precise-proposed main multiverse restricted universe
      deb http://mirror.bjtu.edu.cn/ubuntu/ precise-security main multiverse restricted universe
      deb http://mirror.bjtu.edu.cn/ubuntu/ precise-updates main multiverse restricted universe
      deb-src http://mirror.bjtu.edu.cn/ubuntu/ precise main multiverse restricted universe
      deb-src http://mirror.bjtu.edu.cn/ubuntu/ precise-backports main multiverse restricted universe
      deb-src http://mirror.bjtu.edu.cn/ubuntu/ precise-proposed main multiverse restricted universe
      deb-src http://mirror.bjtu.edu.cn/ubuntu/ precise-security main multiverse restricted universe
      deb-src http://mirror.bjtu.edu.cn/ubuntu/ precise-updates main multiverse restricted universe
      更新源
      apt-get update
      添加openstack的源
      apt-get install python-software-properties add-apt-repository cloud-archive:havana
      更新源，以及更新软件
      apt-get update -y apt-get upgrade -y apt-get dist-upgrade -y
      reboot
    4. 安装NTP服务器来让计算节点节点同步时间
      apt-get install ntp
      apt-get install rsplib-tools
      vi /etc/ntp.conf
      1
      2
      server 127.127.1.0
      fudge 127.127.1.0 stratum 10
      service ntp restart
    5. 安装MySQL server 
      1
      2
      3
      apt-get install python-mysqldb mysql-server
      sed -i 's/127.0.0.1/0.0.0.0/g' /etc/mysql/my.cnf
      /etc/init.d/mysql restart
    6. 开启IP转发
      1
      2
      3
      sed -i -r 's/^\s*#(net\.ipv4\.ip_forward=1.*)/\1/' /etc/sysctl.conf
      echo 1 > /proc/sys/net/ipv4/ip_forward
      sysctl -p
    7. 安装其他服务
      1
      apt-get install -y vlan bridge-utils

  2. 配置

    1. 安装rabbitmq，是控制器统筹各个部件的软件，并修改用户guest的密码为password
      1
      2
      apt-get install rabbitmq-server
      rabbitmqctl change_password guest password
      为每一个部件都建立数据库
      mysql -u root -p
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      	CREATE DATABASE keystone;
      	G RANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'controller' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' IDENTIFIED BY 'password';
      	
      	CREATE DATABASE glance;
      	G RANT ALL PRIVILEGES ON glance.* TO 'glance'@'localhost' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON glance.* TO 'glance'@'controller' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON glance.* TO 'glance'@'%' IDENTIFIED BY 'password';
      	
      	CREATE DATABASE nova;
      	G RANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON nova.* TO 'nova'@'controller' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON nova.* TO 'nova'@'%' IDENTIFIED BY 'password';
      	
      	CREATE DATABASE neutron;
      	G RANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'localhost' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'controller' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'%' IDENTIFIED BY 'password';
      	
      	CREATE DATABASE cinder;
      	G RANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'controller' IDENTIFIED BY 'password';
      	G RANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'%' IDENTIFIED BY 'password';
      quit;
    2. Keystone的安装，是openstack的验证模块
      apt-get install keystone
      修改对应的数据库连接
      vi /etc/keystone/keystone.conf
      1
      connection = mysql://keystone:password@controller/keystone
      删除原始的本地数据库，以免错误调用
      rm /var/lib/keystone/keystone.db
      service keystone restart
      同步数据库
      keystone-manage db_sync
      为各个部件在keystone中添加用户，服务，和后端
      1
      2
      3
      vi keystone.sh
      chmod +x keystone.sh
      ./keystone.sh
      验证keystone
      1
      2
      3
      unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT
      keystone --os-username=admin --os-password=password --os-auth-url=http://controller:35357/v2.0 token-get
      keystone --os-username=admin --os-password=password --os-tenant-name=admin --os-auth-url=http://controller:35357/v2.0 token-get
      导入本地变量
      vi keystonerc
      1
      2
      3
      4
      export OS_TENANT_NAME=admin
      export OS_USERNAME=admin
      export OS_PASSWORD=password
      export OS_AUTH_URL=http://controller_ext:35357/v2.0
      1
      2
      3
      source keystonerc
      keystone token-get
      keystone user-list
    3. Glance的安装，用于存储镜像
      apt-get install glance
      修改数据库连接，rabbit的密码，keystone的验证
      vi /etc/glance/glance-api.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      sql_connection = mysql://glance:password@controller/glance

      #rabbit_password = guest
      rabbit_password = password

      [keystone_authtoken]
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      admin_tenant_name = service
      admin_user = glance
      admin_password = password

      [paste_deploy]
      flavor = keystone
      vi /etc/glance/glance-registry.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      sql_connection = mysql://glance:password@controller/glance

      [keystone_authtoken]
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      admin_tenant_name = service
      admin_user = glance
      admin_password = password

      [paste_deploy]
      flavor = keystone
      删除原来的本地数据库，以免错误调用
      rm /var/lib/glance/glance.sqlite
      修改API调用时的keystone验证信息
      vi /etc/glance/glance-api-paste.ini
      1
      2
      3
      4
      5
      6
      7
      8
      9
      [filter:authtoken]
      paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
      delay_auth_decision = true
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      admin_tenant_name = service
      admin_user = glance
      admin_password = password
      vi /etc/glance/glance-registry-paste.ini
      1
      2
      3
      4
      5
      6
      7
      8
      [filter:authtoken]
      paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      admin_tenant_name = service
      admin_user = glance
      admin_password = password
      重启服务
      1
      2
      service glance-registry restart
      service glance-api restart
      导入数据库
      glance-manage db_sync
      验证glance服务
      1
      2
      3
      mkdir images
      cd images
      wget http://cdn.download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86\_64-disk.img
      导入镜像
      glance image-create --name="CirrOS 0.3.1" --disk-format=qcow2 --container-format=bare --is-public=true < cirros-0.3.1-x86_64-disk.img glance image-list
    4. 安装Neutron
      安装Neutron的主体
      apt-get install neutron-server
      配置rabbit，keystone和数据库的验证
      vi /etc/neutron/neutron.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      [default]
      auth_strategy = keystone

      rpc_backend = neutron.openstack.common.rpc.impl_kombu
      rabbit_host = controller
      rabbit_port = 5672
      rabbit_password = password

      [keystone_authtoken]
      auth_host = controller
      admin_tenant_name = service
      admin_user = neutron
      admin_password = password
      #auth_url = http://controller:35357/v2.0

      [database]
      connection = mysql://neutron:password@controller/neutron
      配置API调用是的验证
      vi /etc/neutron/api-paste.ini
      1
      2
      3
      4
      5
      [filter:authtoken]
      paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
      admin_tenant_name = service
      admin_user = neutron
      admin_password = password
      配置nova部分，之前要先安装nova-api，nova不设置防火墙，只调用neutron的防火墙
      vi /etc/nova/nova.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      network_api_class=nova.network.neutronv2.api.API
      neutron_url=http://controller:9696
      neutron_auth_strategy=keystone
      neutron_admin_tenant_name=service
      neutron_admin_username=neutron
      neutron_admin_password=password
      neutron_admin_auth_url=http://controller:35357/v2.0
      linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
      #firewall_driver=nova.virt.libvirt.firewall.IptablesFirewallDriver
      firewall_driver=nova.virt.firewall.NoopFirewallDriver
      security_group_api=neutron
      安装neutron的插件openvswitch
      apt-get install neutron-plugin-openvswitch-agent
      配置neutron插件，配置GRE模式
      vi /etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      [DATABASE]
      #sql_connection = mysql://neutron:password@controller/neutron
      connection = mysql://neutron:password@controller/neutron
      [OVS]
      tenant_network_type = gre
      tunnel_id_ranges = 1:1000
      enable_tunneling = True

      [SECURITYGROUP]
      firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
    5. 安装Nova
      apt-get install nova-novncproxy novnc nova-api nova-ajax-console-proxy nova-cert nova-conductor nova-consoleauth nova-doc nova-scheduler python-novaclient
      配置rabbit，keystone，metadata，vncserver和数据库的验证
      vi /etc/nova/nova.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      26
      27
      28
      29
      30
      31
      32
      33
      34
      35
      36
      37
      38
      39
      40
      41
      42
      43
      44
      [DEFAULT]
      rpc_backend = nova.rpc.impl_kombu
      rabbit_host = controller
      rabbit_userid = guest
      rabbit_password = password           

      vncserver_proxyclient_address=#management ip
      vncserver_listen=0.0.0.0

      auth_strategy=keystone

      glance_host=controller

      # Network settings
      network_api_class=nova.network.neutronv2.api.API
      neutron_url=http://controller:9696
      neutron_auth_strategy=keystone
      neutron_admin_tenant_name=service
      neutron_admin_username=neutron
      neutron_admin_password=password
      neutron_admin_auth_url=http://controller:35357/v2.0
      linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
      firewall_driver=nova.virt.firewall.NoopFirewallDriver
      security_group_api=neutron

      #Metadata
      service_neutron_metadata_proxy = True
      neutron_metadata_proxy_shared_secret = password
      metadata_host = controller
      metadata_listen = 0.0.0.0
      metadata_listen_port = 8775

      # Auth
      #use_deprecated_auth=false

      [database]
      connection = mysql://nova:password@controller/nova
      [keystone_authtoken]
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      admin_tenant_name = service
      admin_user = nova
      admin_password = password
      配置提供API的keystone验证
      vi /etc/nova/api-paste.ini
      1
      2
      3
      4
      5
      6
      7
      8
      9
      [filter:authtoken]
      paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
      auth_host = controller
      auth_port = 35357
      auth_protocol = http
      auth_uri = http://controller:5000/v2.0
      admin_tenant_name = service
      admin_user = nova
      admin_password = password
      同步数据库
      1
      2
      3
      4
      5
      6
      7
      8
      nova-manage db sync

      service nova-api restart
      service nova-cert restart
      service nova-consoleauth restart
      service nova-scheduler restart
      service nova-conductor restart
      service nova-novncproxy restart
      查看服务运行的状态
      nova-manage service list
      修改默认的quota
      vi /etc/nova/nova.conf
      1
      2
      3
      quota_instances=100
      quota_cores=1000
      quota_ram=1048576
      1
      2
      3
      4
      5
      6
      7
      8
      9
      service nova-api restart
      service nova-cert restart
      service nova-consoleauth restart
      service nova-scheduler restart
      service nova-conductor restart
      service nova-novncproxy restart
      nova-manage db sync

      nova-manage project quota admin
    6. 安装horizon界面
      apt-get install memcached libapache2-mod-wsgi openstack-dashboard
      去掉Ubuntu相关的Openstack主题，bug
      apt-get remove --purge openstack-dashboard-ubuntu-theme
      vi /etc/apache2/httpd.conf
      1
      ServerName controller_ext
      1
      2
      service apache2 restart
      service memcached restart
    7. 安装cinder，用于提供虚拟存储
      apt-get install cinder-api cinder-scheduler
      配置cinder数据库
      vi /etc/cinder/cinder.conf
      1
      2
      3
      4
      5
      [database]
      ...
      # The SQLAlchemy connection string used to connect to the
      # database (string value)
      connection = mysql://cinder:password@controller/cinder
      cinder-manage db sync
      配置keystone的验证
      vi /etc/cinder/api-paste.ini
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      [filter:authtoken]
      paste.filter_factory=keystoneclient.middleware.auth_token:filter_factory
      auth_host=controller
      auth_port = 35357
      auth_protocol = http
      auth_uri = http://controller:5000/v2.0
      admin_tenant_name=service
      admin_user=cinder
      admin_password=password
      #auth_version = v2.0
      vi /etc/cinder/cinder.conf
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      [DEFAULT]
      rpc_backend = cinder.openstack.common.rpc.impl_kombu
      rabbit_host = controller
      rabbit_port = 5672
      rabbit_userid = guest
      rabbit_password = password

      glance_host=controller
      glance_port=9292
      glance_api_servers=controller:9292
      1
      2
      service cinder-scheduler restart
      service cinder-api restart

网络节点：

由于网络节点配置在控制节点上，所以第一部的准备工作就可以去掉
开启转发
vi /etc/sysctl.conf

1
2
3
4
net.ipv4.ip_forward=1
#小心配错
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
1
2
sysctl -p
service networking restart
  1. 安装neutron的各种网络服务
    apt-get install neutron-server neutron-dhcp-agent neutron-plugin-openvswitch-agent neutron-l3-agent
    配置验证信息
    vi /etc/neutron/neutron.conf
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    [default]
    auth_strategy = keystone

    rpc_backend = neutron.openstack.common.rpc.impl_kombu
    rabbit_host = controller
    rabbit_port = 5672
    rabbit_userid = guest
    rabbit_password = password

    [keystone_authtoken]
    auth_host = controller
    auth_port = 35357
    auth_protocol = http
    admin_tenant_name = service
    admin_user = neutron
    admin_password = password

    [database]
    connection = mysql://neutron:password@controller/neutron
    配置提供API的验证信息
    vi /etc/neutron/api-paste.ini
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    [filter:authtoken]
    paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
    auth_host=controller
    auth_port = 35357
    auth_protocol = http
    auth_uri = http://controller:5000/v2.0
    admin_tenant_name = service
    admin_user = neutron
    admin_password = password
    #page 73
    指定提供dhcp的驱动服务
    vi /etc/neutron/dhcp_agent.ini
    1
    dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
    vi /etc/nova/nova.conf
    1
    2
    3
    4
    5
    6
    [DEFAULT]
    service_neutron_metadata_proxy = true
    neutron_metadata_proxy_shared_secret = password
    metadata_host = controller
    metadata_listen = 0.0.0.0
    metadata_listen_port = 8775
    1
    service nova-api restart
    配置meta的验证服务
    vi /etc/neutron/metadata_agent.ini
    1
    2
    3
    4
    5
    6
    7
    8
    [DEFAULT]
    auth_url = http://controller:5000/v2.0
    auth_region = RegionOne
    admin_tenant_name = service
    admin_user = neutron
    admin_password = password
    nova_metadata_ip = controller
    metadata_proxy_shared_secret = password
    1
    2
    3
    4
    5
    service neutron-server restart
    service neutron-dhcp-agent restart
    service neutron-l3-agent restart
    service neutron-metadata-agent restart
    service neutron-plugin-openvswitch-agent restart
    安装neutron的插件openvswtich
    1
    2
    3
    4
    apt-get install neutron-plugin-openvswitch-agent openvswitch-switch
    service openvswitch-switch restart
    ovs-vsctl add-br br-int
    ovs-vsctl add-br br-ex
    把提供外部API的网口桥接到br-ex上
    ovs-vsctl add-port br-ex eth0
    修改网卡配置，把eth0改成混杂模式，把br-ex改成eth0的ip配置
    vi /etc/network/interfaces
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    #For Exposing OpenStack API over the internet
    auto eth0
    iface eth0 inet manual
    up ifconfig eth0 0.0.0.0 up
    up ip link set eth0 promisc on
    down ip link set eth0 promisc off
    down ifconfig eth0 down

    auto br-ex
    iface br-ex inet static
    address 192.168.1.10
    netmask 255.255.255.0
    gateway 192.168.1.1
    /etc/init.d/networking restart
    配置l3_agent和dhcp_agent
    vi /etc/neutron/l3_agent.ini
    1
    2
    interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
    use_namespaces = True
    vi /etc/neutron/dhcp_agent.ini
    1
    2
    interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
    use_namespaces = True
    配置neutron的插件为openvswitch
    vi /etc/neutron/neutron.conf
    1
    core_plugin = neutron.plugins.openvswitch.ovs_neutron_plugin.OVSNeutronPluginV2
    service neutron-plugin-openvswitch-agent restart
    配置插件为GRE模式
    vi /etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    [OVS]
    tenant_network_type = gre
    tunnel_id_ranges = 1:1000
    enable_tunneling = True
    integration_bridge = br-int
    tunnel_bridge = br-tun
    local_ip = #tunnel ip

    [securitygroup]
    # Firewall driver for realizing neutron security group function.
    firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
    service neutron-plugin-openvswitch-agent restart