CoreOS + etcd2 + Flannel ......for Docker－echochio

抓 IOS ...
用這 IOS 開機 ..

直接安裝 OS (我用 1298.6.0 )

sudo su - root
coreos-install -d /dev/sda -C stable -c ~/cloud-config.yaml

三台 cloud-config.yaml 檔案 ....

執行 curl -w "\n" 'https://discovery.etcd.io/new?size=1' 會產生 discovery:
https://discovery.etcd.io/0dd2252d1ca6eceb3fe05cfd789ceee9

三台的 cloud-config.yaml 差在 hostname & node name & IP 其他都一樣
其中 "Network": "172.17.0.0/16" 是 虛擬IP地址區段 可自行修改的

可用
wget https://raw.githubusercontent.com/chio-nzgft/CoreOS-cloud-config/master/cloud-config-node1.yaml
wget https://raw.githubusercontent.com/chio-nzgft/CoreOS-cloud-config/master/cloud-config-node2.yaml
wget https://raw.githubusercontent.com/chio-nzgft/CoreOS-cloud-config/master/cloud-config-node3.yaml

抓這三個 node 的 cloud-config

#cloud-config
hostname: coreos1
ssh_authorized_keys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAs9jMlJJ..................M+CsRy+z8PzPQ== echochio

coreos:
  units:
    - name: etcd2.service
      command: start
    - name: systemd-networkd.service
      command: stop
    - name: 00-eth0.network
      runtime: true
      content: |
        [Match]
        Name=eth0

        [Network]
        DNS=168.95.1.1
        Address=192.168.0.69/24
        Gateway=192.168.0.2
    - name: down-interfaces.service
      command: start
      content: |
        [Service]
        Type=oneshot
        ExecStart=/usr/bin/ip link set eth0 down
        ExecStart=/usr/bin/ip addr flush dev eth0
    - name: systemd-networkd.service
      command: restart
    - name: settimezone.service
      command: start
      content: |
        [Unit]
        Description=Set the time zone

        [Service]
        ExecStart=/usr/bin/timedatectl set-timezone  Asia/Taipei
        RemainAfterExit=yes
        Type=oneshot

  etcd2:
    name: "node01"
    discovery: https://discovery.etcd.io/9555056f87b2eec75214dc27703e5add
    advertise-client-urls: http://192.168.0.69:2379
    initial-advertise-peer-urls: http://192.168.0.69:2380
    listen-client-urls: http://0.0.0.0:2379
    listen-peer-urls: http://192.168.0.69:2380

  flannel:
    interface: 192.168.0.69

  units:
    - name: etcd2.service
      command: start
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Service]
            Restart=always

            [Install]
            WantedBy=multi-user.target

    - name: flanneld.service
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Unit]
            After=etcd2.service
            Requires=etcd2.service

            [Service]
            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "172.17.0.0/16" }'

            [Install]
            WantedBy=multi-user.target
      command: start

    - name: docker.service
      command: start
      drop-ins:
      - name: 60-docker-wait-for-flannel-config.conf
        content: |
          [Unit]
          After=flanneld.service
          Requires=flanneld.service

          [Service]
          Restart=always

          [Install]
          WantedBy=multi-user.target


    - name: docker-tcp.socket
      command: start
      enable: true
      content: |
        [Unit]
        Description=Docker Socket for the API

        [Socket]
        ListenStream=2375
        Service=docker.service

        [Install]
        WantedBy=sockets.target

#cloud-config
hostname: coreos2
ssh_authorized_keys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAs9jMlJJ..................M+CsRy+z8PzPQ== echochio

coreos:
  units:
    - name: etcd2.service
      command: start
    - name: systemd-networkd.service
      command: stop
    - name: 00-eth0.network
      runtime: true
      content: |
        [Match]
        Name=eth0

        [Network]
        DNS=168.95.1.1
        Address=192.168.0.70/24
        Gateway=192.168.0.2
    - name: down-interfaces.service
      command: start
      content: |
        [Service]
        Type=oneshot
        ExecStart=/usr/bin/ip link set eth0 down
        ExecStart=/usr/bin/ip addr flush dev eth0
    - name: systemd-networkd.service
      command: restart
    - name: settimezone.service
      command: start
      content: |
        [Unit]
        Description=Set the time zone

        [Service]
        ExecStart=/usr/bin/timedatectl set-timezone  Asia/Taipei
        RemainAfterExit=yes
        Type=oneshot

  etcd2:
    name: "node02"
    discovery: https://discovery.etcd.io/9555056f87b2eec75214dc27703e5add
    advertise-client-urls: http://192.168.0.70:2379
    initial-advertise-peer-urls: http://192.168.0.70:2380
    listen-client-urls: http://0.0.0.0:2379
    listen-peer-urls: http://192.168.0.70:2380

  flannel:
    interface: 192.168.0.70

  units:
    - name: etcd2.service
      command: start
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Service]
            Restart=always

            [Install]
            WantedBy=multi-user.target

    - name: flanneld.service
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Unit]
            After=etcd2.service
            Requires=etcd2.service

            [Service]
            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "172.17.0.0/16" }'

            [Install]
            WantedBy=multi-user.target
      command: start

    - name: docker.service
      command: start
      drop-ins:
      - name: 60-docker-wait-for-flannel-config.conf
        content: |
          [Unit]
          After=flanneld.service
          Requires=flanneld.service

          [Service]
          Restart=always

          [Install]
          WantedBy=multi-user.target


    - name: docker-tcp.socket
      command: start
      enable: true
      content: |
        [Unit]
        Description=Docker Socket for the API

        [Socket]
        ListenStream=2375
        Service=docker.service

        [Install]
        WantedBy=sockets.target

#cloud-config
hostname: coreos3
ssh_authorized_keys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAs9jMlJJ..................M+CsRy+z8PzPQ== echochio

coreos:
  units:
    - name: etcd2.service
      command: start
    - name: systemd-networkd.service
      command: stop
    - name: 00-eth0.network
      runtime: true
      content: |
        [Match]
        Name=eth0

        [Network]
        DNS=168.95.1.1
        Address=192.168.0.71/24
        Gateway=192.168.0.2
    - name: down-interfaces.service
      command: start
      content: |
        [Service]
        Type=oneshot
        ExecStart=/usr/bin/ip link set eth0 down
        ExecStart=/usr/bin/ip addr flush dev eth0
    - name: systemd-networkd.service
      command: restart
    - name: settimezone.service
      command: start
      content: |
        [Unit]
        Description=Set the time zone

        [Service]
        ExecStart=/usr/bin/timedatectl set-timezone  Asia/Taipei
        RemainAfterExit=yes
        Type=oneshot

  etcd2:
    name: "node03"
    discovery: https://discovery.etcd.io/9555056f87b2eec75214dc27703e5add
    advertise-client-urls: http://192.168.0.71:2379
    initial-advertise-peer-urls: http://192.168.0.71:2380
    listen-client-urls: http://0.0.0.0:2379
    listen-peer-urls: http://192.168.0.71:2380

  flannel:
    interface: 192.168.0.71

  units:
    - name: etcd2.service
      command: start
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Service]
            Restart=always

            [Install]
            WantedBy=multi-user.target

    - name: flanneld.service
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Unit]
            After=etcd2.service
            Requires=etcd2.service

            [Service]
            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "172.17.0.0/16" }'

            [Install]
            WantedBy=multi-user.target
      command: start

    - name: docker.service
      command: start
      drop-ins:
      - name: 60-docker-wait-for-flannel-config.conf
        content: |
          [Unit]
          After=flanneld.service
          Requires=flanneld.service

          [Service]
          Restart=always

          [Install]
          WantedBy=multi-user.target


    - name: docker-tcp.socket
      command: start
      enable: true
      content: |
        [Unit]
        Description=Docker Socket for the API

        [Socket]
        ListenStream=2375
        Service=docker.service

        [Install]
        WantedBy=sockets.target

不知為何 IP 都無法固定 ...最後進去用手動設 /etc/systemd/network/static.network
應該是 cloud-config 沒寫好 ....
原文是說 :

即使您已經設置了靜態IP地址，並且您的網絡中有一個可用的DHCP服務器，則systemd-networkd仍將使用DHCP分配IP地址

修改好了會自動設定上去了 ....原來是要先停用再啟用 ,,,,時區也加上去了

剛開始只有一台 etcd上線
core@coreos1 ~ $ etcdctl cluster-health
member 74a1e2b4597f8a15 is healthy: got healthy result from http://192.168.0.69:2379
cluster is healthy

加第二台
core@coreos1 ~ $ etcdctl member add node02 http://192.168.0.70:2380
Added member named node02 with ID b15622694b806753 to cluster

加完後到第二台重啟 :
先編輯檔案
core@coreos2 ~ $ sudo vi /run/systemd/system/etcd2.service.d/99-restore.conf

[Service]
ExecStartPre=/usr/bin/rm -rf /var/lib/etcd2/proxy
Environment="ETCD_DISCOVERY="
Environment="ETCD_NAME=node02"
Environment="ETCD_INITIAL_CLUSTER=node01=http://192.168.0.69:2380,node02=http://192.168.0.70:2380"
Environment="ETCD_INITIAL_CLUSTER_STATE=existing"

重啟 :
core@coreos2 ~ $ sudo systemctl daemon-reload
core@coreos2 ~ $ sudo systemctl restart etcd2

查看正常了
core@coreos1 ~ $ etcdctl cluster-health
member 5e88ef78488df9d1 is healthy: got healthy result from http://192.168.0.70:2379
member b15622694b806753 is healthy: got healthy result from http://192.168.0.69:2379
cluster is healthy

coreos1 ~ #  etcdctl member list
5e88ef78488df9d1: name=node02 peerURLs=http://192.168.0.70:2380 clientURLs=http://192.168.0.70:2379 isLeader=false
b15622694b806753: name=node01 peerURLs=http://192.168.0.69:2380 clientURLs=http://192.168.0.69:2379 isLeader=true

第三台同樣方式處理後 ....

移除一台 node 比較簡單 ...例如 node2 ...5e88ef78488df9d1

etcdctl member remove 5e88ef78488df9d1

設定Flannel

到每台 node 查看是否 docker0 ip 與 flannel0 相同
且 etcdctl 的 /coreos.com/network 看的到每台 IP 有就是 cloud-config.yaml
內有寫進去啟動 Flannel

coreos3 system # ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 172.17.69.1  netmask 255.255.255.0  broadcast 0.0.0.0
        ether 02:42:9b:86:44:8c  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.0.71  netmask 255.255.255.0  broadcast 192.168.0.255
        inet6 fe80::215:5dff:fe00:3010  prefixlen 64  scopeid 0x20<link>
        ether 00:15:5d:00:30:10  txqueuelen 1000  (Ethernet)
        RX packets 476195  bytes 61316767 (58.4 MiB)
        RX errors 0  dropped 1179  overruns 0  frame 0
        TX packets 285224  bytes 31406803 (29.9 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel0: flags=4305<UP,POINTOPOINT,RUNNING,NOARP,MULTICAST>  mtu 1472
        inet 172.17.69.0  netmask 255.255.0.0  destination 172.17.69.0
        inet6 fe80::25f9:b8db:197b:3093  prefixlen 64  scopeid 0x20<link>
        unspec 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00  txqueuelen 500  (UNSPEC)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 8  bytes 384 (384.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 73979  bytes 14720247 (14.0 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 73979  bytes 14720247 (14.0 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

coreos3 system # etcdctl ls /coreos.com/network/subnets
/coreos.com/network/subnets/172.17.69.0-24
/coreos.com/network/subnets/172.17.77.0-24
/coreos.com/network/subnets/172.17.35.0-24
coreos3 system # etcdctl get /coreos.com/network/subnets/172.17.69.0-24
{"PublicIP":"192.168.0.71"}
coreos3 system # etcdctl get /coreos.com/network/subnets/172.17.77.0-24
{"PublicIP":"192.168.0.69"}
coreos3 system # etcdctl get /coreos.com/network/subnets/172.17.35.0-24
{"PublicIP":"192.168.0.70"}

CoreOS 已不用 fleet .... 雖然參考下面設定起來還是不要用吧 !!
....將它關閉吧 !!!!

systemctl disable fleet
systemctl stop fleet

--------------------------------------------------------------------------

三台都做 fleet 啟動

coreos1 ~ # systemctl enable fleet
Created symlink /etc/systemd/system/multi-user.target.wants/fleet.service → /usr/lib/systemd/system/fleet.service.
coreos1 ~ # systemctl start fleet
coreos1 ~ # systemctl status fleet
● fleet.service - fleet daemon
   Loaded: loaded (/usr/lib/systemd/system/fleet.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2017-03-29 03:45:52 UTC; 1s ago
 Main PID: 1345 (fleetd)
    Tasks: 6
   Memory: 2.3M
      CPU: 10ms
   CGroup: /system.slice/fleet.service
           └─1345 /usr/bin/fleetd

Mar 29 03:45:52 coreos1 systemd[1]: Started fleet daemon.
Mar 29 03:45:52 coreos1 fleetd[1345]: INFO fleetd.go:64: Starting fleetd version 0.11.8
Mar 29 03:45:52 coreos1 fleetd[1345]: INFO fleetd.go:170: No provided or default config file found - proceeding without
Mar 29 03:45:52 coreos1 fleetd[1345]: INFO server.go:160: Establishing etcd connectivity
Mar 29 03:45:53 coreos1 fleetd[1345]: INFO server.go:171: Starting server components
Mar 29 03:45:53 coreos1 fleetd[1345]: INFO engine.go:79: Engine leader is c71d347291fe4a75bb5e07badbdb9b0a

最後啟動完後看到

coreos1 ~ #  fleetctl list-machines
MACHINE         IP              METADATA
90573c4f...     192.168.0.69    -
9ebc7717...     192.168.0.71    -
c71d3472...     192.168.0.70    -
coreos1 ~ # fleetctl list-units
UNIT    MACHINE ACTIVE  SUB

發生 ........etcdctl member list 狀態也不正常

到壞的那台去處理先將服務停掉 (我的是 node1) :
systemctl stop etcd2

將壞的 DB 刪除
rm -rf /var/lib/etcd2/member/

到好的 coreos2 那台將 node1 刪除

etcdctl member remove b15622694b806753

重建連接檔
core@coreos1 ~ $ sudo vi /run/systemd/system/etcd2.service.d/99-restore.conf

[Service]
ExecStartPre=/usr/bin/rm -rf /var/lib/etcd2/proxy
Environment="ETCD_DISCOVERY="
Environment="ETCD_NAME=node01"
Environment="ETCD_INITIAL_CLUSTER=node02=http://192.168.0.70:2380,node03=http://192.168.0.71:2380,node01=http://192.168.0.69:2380"
Environment="ETCD_INITIAL_CLUSTER_STATE=existing"

到好的機器coreos2 加回第一台
core@coreos2 ~ $ etcdctl member add node01 http://192.168.0.69:2380

加完後到第一台重啟 etcd2 :

重啟 :
core@coreos1 ~ $ sudo systemctl daemon-reload
core@coreos1 ~ $ sudo systemctl restart etcd2

查看 etcdctl cluster-health 是否三台都上線了

啟動 fleet
systemctl start fleet
systemctl status fleet

查看 fleetctl 是否三台都慢慢上線了
fleetctl list-machines

如壞兩台那只能用災難恢原 .....

https://coreos.com/etcd/docs/latest/etcd-live-cluster-reconfiguration.html#etcd-disaster-recovery-on-coreos

echochio

echochio 發表在痞客邦留言(0) 人氣()

echochio

echochio的部落格跳到主文歡迎光臨echochio在痞客邦的小天地，我是一個(C/P)收集工程師^ ^ 所以想要跟(C/P)同好收集內容因此有需要(C/P)下面程式語法的同好請來和我交流, 分享相片

CoreOS + etcd2 + Flannel ......for Docker

歷史上的今天

留言列表

站方公告

活動快報

愛睡噴...

我的好友

熱門文章

文章分類

雜七雜八 (1)

程式 (5)

最新文章

最新留言

動態訂閱

文章精選

文章搜尋

新聞交換(RSS)

誰來我家

參觀人氣

QR Code

POWERED BY

echochio

echochio的部落格跳到主文歡迎光臨echochio在痞客邦的小天地 ，我是一個(C/P)收集工程師^ ^ 所以想要跟(C/P)同好收集內容 因此有需要(C/P)下面程式語法的同好請來和我交流, 分享相片

CoreOS + etcd2 + Flannel ......for Docker

歷史上的今天

留言列表

站方公告

活動快報

愛睡噴...

我的好友

熱門文章

文章分類

雜七雜八 (1)

程式 (5)

最新文章

最新留言

動態訂閱

文章精選

文章搜尋

新聞交換(RSS)

誰來我家

參觀人氣

QR Code

POWERED BY

echochio的部落格跳到主文歡迎光臨echochio在痞客邦的小天地，我是一個(C/P)收集工程師^ ^ 所以想要跟(C/P)同好收集內容因此有需要(C/P)下面程式語法的同好請來和我交流, 分享相片