Friday, June 24, 2016

Elasticsearch migrates data and configuration to docker host directory


Method 1

$docker cp [OPTIONS] CONTAINER:SRC_PATH DEST_PATH

Connect to a running Elasticsearch container

ubuntu@node1:~$ docker ps
CONTAINER ID        IMAGE                 COMMAND                  CREATED             STATUS              PORTS                                                NAMES
d353dbbe21d3        kibana:4.5.1          "/docker-entrypoint.s"   2 hours ago         Up 2 hours          10.0.2.41:5601->5601/tcp                             kibana-es
f3dbc0e04fc7        elasticsearch:2.3.3   "/docker-entrypoint.s"   2 hours ago         Up 37 seconds       10.0.2.41:9200->9200/tcp, 10.0.2.41:9300->9300/tcp   esearch
ubuntu@node1:~$ 
ubuntu@node1:~$ docker exec -it --user elasticsearch esearch bash
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ id                                                                                       
uid=105(elasticsearch) gid=108(elasticsearch) groups=108(elasticsearch)
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$

Default path of config, data and logs are "/usr/share/elasticsearch"

elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ tar cf /tmp/data.tar data
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ tar cf /tmp/config.tar config
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ tar cf /tmp/logs.tar logs    
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ ls -l /tmp/*.tar
-rw-r--r-- 1 elasticsearch elasticsearch  10240 Jun 24 12:47 /tmp/config.tar
-rw-r--r-- 1 elasticsearch elasticsearch 614400 Jun 24 12:46 /tmp/data.tar
-rw-r--r-- 1 elasticsearch elasticsearch  10240 Jun 24 12:47 /tmp/logs.tar
elasticsearch@f3dbc0e04fc7:/usr/share/elasticsearch$ exit


ubuntu@node1:~/docker/elasticsearch/backup$ docker exec esearch bash -c "ls -l /tmp/*.tar"
-rw-r--r-- 1 elasticsearch elasticsearch  10240 Jun 24 12:47 /tmp/config.tar
-rw-r--r-- 1 elasticsearch elasticsearch 614400 Jun 24 12:46 /tmp/data.tar
-rw-r--r-- 1 elasticsearch elasticsearch  10240 Jun 24 12:47 /tmp/logs.tar
ubuntu@node1:~/docker/elasticsearch/backup$ 
ubuntu@node1:~/docker/elasticsearch/backup$ docker cp esearch:/tmp/config.tar .
ubuntu@node1:~/docker/elasticsearch/backup$ docker cp esearch:/tmp/data.tar .
ubuntu@node1:~/docker/elasticsearch/backup$ docker cp esearch:/tmp/logs.tar .
ubuntu@node1:~/docker/elasticsearch/backup$ ls -l *.tar
-rw-r--r-- 1 ubuntu ubuntu  10240 Jun 24 19:47 config.tar
-rw-r--r-- 1 ubuntu ubuntu 614400 Jun 24 19:46 data.tar
-rw-r--r-- 1 ubuntu ubuntu  10240 Jun 24 19:47 logs.tar


Method 2 method with a tar command:

$docker exec foo tar Ccf $(dirname SRC_PATH) - $(basename SRC_PATH) | tar Cxf DEST_PATH -

Since tar options mean:
  • C -> Change to directory
  • c -> Create a new archive
  • f -> Specific archive file or device
  • x -> Extract archive file or device
ubuntu@node1:~/docker/elasticsearch/backup$ docker exec esearch tar Ccf /usr/share/elasticsearch - config | tar Cxf . -
ubuntu@node1:~/docker/elasticsearch/backup$ 
ubuntu@node1:~/docker/elasticsearch/backup$ docker exec esearch tar Ccf /usr/share/elasticsearch - data | tar Cxf . -
ubuntu@node1:~/docker/elasticsearch/backup$ 
ubuntu@node1:~/docker/elasticsearch/backup$ docker exec esearch tar Ccf /usr/share/elasticsearch - logs | tar Cxf . -
ubuntu@node1:~/docker/elasticsearch/backup$ 
ubuntu@node1:~/docker/elasticsearch/backup$ ls -l
total 12
drwxr-xr-x 3 ubuntu ubuntu 4096 Jun  9 21:58 config
drwxr-xr-x 3 ubuntu ubuntu 4096 Jun 24 12:08 data
drwxr-xr-x 2 ubuntu ubuntu 4096 Jun  9 21:58 logs


Restore

Restore to host directory

ubuntu@node1:~/docker/elasticsearch/backup$ cd ..
ubuntu@node1:~/docker/elasticsearch$ 
ubuntu@node1:~/docker/elasticsearch$ tar xvf backup/config.tar 
config/
config/logging.yml
config/elasticsearch.yml
config/scripts/
ubuntu@node1:~/docker/elasticsearch$ tar xvf backup/data.tar 
data/
data/elasticsearch/
data/elasticsearch/nodes/
data/elasticsearch/nodes/0/
data/elasticsearch/nodes/0/indices/
data/elasticsearch/nodes/0/indices/.kibana/
data/elasticsearch/nodes/0/indices/.kibana/0/
data/elasticsearch/nodes/0/indices/.kibana/0/index/
data/elasticsearch/nodes/0/indices/.kibana/0/index/segments_5
data/elasticsearch/nodes/0/indices/.kibana/0/index/write.lock
data/elasticsearch/nodes/0/indices/.kibana/0/index/_1.cfs
data/elasticsearch/nodes/0/indices/.kibana/0/index/_1.si
data/elasticsearch/nodes/0/indices/.kibana/0/index/_1.cfe
data/elasticsearch/nodes/0/indices/.kibana/0/_state/
data/elasticsearch/nodes/0/indices/.kibana/0/_state/state-3.st
data/elasticsearch/nodes/0/indices/.kibana/0/translog/
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog-4.ckp
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog.ckp
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog-5.tlog
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/.kibana/0/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/.kibana/_state/
data/elasticsearch/nodes/0/indices/.kibana/_state/state-4.st
data/elasticsearch/nodes/0/indices/bank/
data/elasticsearch/nodes/0/indices/bank/3/
data/elasticsearch/nodes/0/indices/bank/3/index/
data/elasticsearch/nodes/0/indices/bank/3/index/_0.cfs
data/elasticsearch/nodes/0/indices/bank/3/index/_0.cfe
data/elasticsearch/nodes/0/indices/bank/3/index/segments_5
data/elasticsearch/nodes/0/indices/bank/3/index/_0.si
data/elasticsearch/nodes/0/indices/bank/3/index/write.lock
data/elasticsearch/nodes/0/indices/bank/3/_state/
data/elasticsearch/nodes/0/indices/bank/3/_state/state-2.st
data/elasticsearch/nodes/0/indices/bank/3/translog/
data/elasticsearch/nodes/0/indices/bank/3/translog/translog.ckp
data/elasticsearch/nodes/0/indices/bank/3/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/bank/3/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/bank/3/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/bank/3/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/bank/3/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/bank/4/
data/elasticsearch/nodes/0/indices/bank/4/index/
data/elasticsearch/nodes/0/indices/bank/4/index/_0.cfs
data/elasticsearch/nodes/0/indices/bank/4/index/_0.cfe
data/elasticsearch/nodes/0/indices/bank/4/index/segments_5
data/elasticsearch/nodes/0/indices/bank/4/index/_0.si
data/elasticsearch/nodes/0/indices/bank/4/index/write.lock
data/elasticsearch/nodes/0/indices/bank/4/_state/
data/elasticsearch/nodes/0/indices/bank/4/_state/state-2.st
data/elasticsearch/nodes/0/indices/bank/4/translog/
data/elasticsearch/nodes/0/indices/bank/4/translog/translog.ckp
data/elasticsearch/nodes/0/indices/bank/4/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/bank/4/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/bank/4/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/bank/4/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/bank/4/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/bank/1/
data/elasticsearch/nodes/0/indices/bank/1/index/
data/elasticsearch/nodes/0/indices/bank/1/index/_0.cfs
data/elasticsearch/nodes/0/indices/bank/1/index/_0.cfe
data/elasticsearch/nodes/0/indices/bank/1/index/segments_5
data/elasticsearch/nodes/0/indices/bank/1/index/_0.si
data/elasticsearch/nodes/0/indices/bank/1/index/write.lock
data/elasticsearch/nodes/0/indices/bank/1/_state/
data/elasticsearch/nodes/0/indices/bank/1/_state/state-2.st
data/elasticsearch/nodes/0/indices/bank/1/translog/
data/elasticsearch/nodes/0/indices/bank/1/translog/translog.ckp
data/elasticsearch/nodes/0/indices/bank/1/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/bank/1/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/bank/1/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/bank/1/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/bank/1/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/bank/0/
data/elasticsearch/nodes/0/indices/bank/0/index/
data/elasticsearch/nodes/0/indices/bank/0/index/_0.cfs
data/elasticsearch/nodes/0/indices/bank/0/index/_0.cfe
data/elasticsearch/nodes/0/indices/bank/0/index/segments_5
data/elasticsearch/nodes/0/indices/bank/0/index/_0.si
data/elasticsearch/nodes/0/indices/bank/0/index/write.lock
data/elasticsearch/nodes/0/indices/bank/0/_state/
data/elasticsearch/nodes/0/indices/bank/0/_state/state-2.st
data/elasticsearch/nodes/0/indices/bank/0/translog/
data/elasticsearch/nodes/0/indices/bank/0/translog/translog.ckp
data/elasticsearch/nodes/0/indices/bank/0/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/bank/0/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/bank/0/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/bank/0/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/bank/0/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/bank/_state/
data/elasticsearch/nodes/0/indices/bank/_state/state-3.st
data/elasticsearch/nodes/0/indices/bank/2/
data/elasticsearch/nodes/0/indices/bank/2/index/
data/elasticsearch/nodes/0/indices/bank/2/index/_0.cfs
data/elasticsearch/nodes/0/indices/bank/2/index/_0.cfe
data/elasticsearch/nodes/0/indices/bank/2/index/segments_5
data/elasticsearch/nodes/0/indices/bank/2/index/_0.si
data/elasticsearch/nodes/0/indices/bank/2/index/write.lock
data/elasticsearch/nodes/0/indices/bank/2/_state/
data/elasticsearch/nodes/0/indices/bank/2/_state/state-2.st
data/elasticsearch/nodes/0/indices/bank/2/translog/
data/elasticsearch/nodes/0/indices/bank/2/translog/translog.ckp
data/elasticsearch/nodes/0/indices/bank/2/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/bank/2/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/bank/2/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/bank/2/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/bank/2/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/customer/
data/elasticsearch/nodes/0/indices/customer/3/
data/elasticsearch/nodes/0/indices/customer/3/index/
data/elasticsearch/nodes/0/indices/customer/3/index/_4.cfe
data/elasticsearch/nodes/0/indices/customer/3/index/_4.cfs
data/elasticsearch/nodes/0/indices/customer/3/index/_4.si
data/elasticsearch/nodes/0/indices/customer/3/index/segments_7
data/elasticsearch/nodes/0/indices/customer/3/index/write.lock
data/elasticsearch/nodes/0/indices/customer/3/_state/
data/elasticsearch/nodes/0/indices/customer/3/_state/state-2.st
data/elasticsearch/nodes/0/indices/customer/3/translog/
data/elasticsearch/nodes/0/indices/customer/3/translog/translog-4.ckp
data/elasticsearch/nodes/0/indices/customer/3/translog/translog.ckp
data/elasticsearch/nodes/0/indices/customer/3/translog/translog-5.tlog
data/elasticsearch/nodes/0/indices/customer/3/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/customer/3/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/customer/3/translog/translog-3.ckp
data/elasticsearch/nodes/0/indices/customer/4/
data/elasticsearch/nodes/0/indices/customer/4/index/
data/elasticsearch/nodes/0/indices/customer/4/index/segments_4
data/elasticsearch/nodes/0/indices/customer/4/index/write.lock
data/elasticsearch/nodes/0/indices/customer/4/_state/
data/elasticsearch/nodes/0/indices/customer/4/_state/state-2.st
data/elasticsearch/nodes/0/indices/customer/4/translog/
data/elasticsearch/nodes/0/indices/customer/4/translog/translog.ckp
data/elasticsearch/nodes/0/indices/customer/4/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/customer/4/translog/translog-1.ckp
data/elasticsearch/nodes/0/indices/customer/4/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/customer/4/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/customer/4/translog/translog-1.tlog
data/elasticsearch/nodes/0/indices/customer/1/
data/elasticsearch/nodes/0/indices/customer/1/index/
data/elasticsearch/nodes/0/indices/customer/1/index/segments_4
data/elasticsearch/nodes/0/indices/customer/1/index/write.lock
data/elasticsearch/nodes/0/indices/customer/1/_state/
data/elasticsearch/nodes/0/indices/customer/1/_state/state-2.st
data/elasticsearch/nodes/0/indices/customer/1/translog/
data/elasticsearch/nodes/0/indices/customer/1/translog/translog.ckp
data/elasticsearch/nodes/0/indices/customer/1/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/customer/1/translog/translog-1.ckp
data/elasticsearch/nodes/0/indices/customer/1/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/customer/1/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/customer/1/translog/translog-1.tlog
data/elasticsearch/nodes/0/indices/customer/0/
data/elasticsearch/nodes/0/indices/customer/0/index/
data/elasticsearch/nodes/0/indices/customer/0/index/segments_4
data/elasticsearch/nodes/0/indices/customer/0/index/write.lock
data/elasticsearch/nodes/0/indices/customer/0/_state/
data/elasticsearch/nodes/0/indices/customer/0/_state/state-2.st
data/elasticsearch/nodes/0/indices/customer/0/translog/
data/elasticsearch/nodes/0/indices/customer/0/translog/translog.ckp
data/elasticsearch/nodes/0/indices/customer/0/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/customer/0/translog/translog-1.ckp
data/elasticsearch/nodes/0/indices/customer/0/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/customer/0/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/customer/0/translog/translog-1.tlog
data/elasticsearch/nodes/0/indices/customer/_state/
data/elasticsearch/nodes/0/indices/customer/_state/state-4.st
data/elasticsearch/nodes/0/indices/customer/2/
data/elasticsearch/nodes/0/indices/customer/2/index/
data/elasticsearch/nodes/0/indices/customer/2/index/_0.cfs
data/elasticsearch/nodes/0/indices/customer/2/index/_0.cfe
data/elasticsearch/nodes/0/indices/customer/2/index/segments_5
data/elasticsearch/nodes/0/indices/customer/2/index/_0.si
data/elasticsearch/nodes/0/indices/customer/2/index/write.lock
data/elasticsearch/nodes/0/indices/customer/2/index/_1.cfs
data/elasticsearch/nodes/0/indices/customer/2/index/_1.si
data/elasticsearch/nodes/0/indices/customer/2/index/_1.cfe
data/elasticsearch/nodes/0/indices/customer/2/_state/
data/elasticsearch/nodes/0/indices/customer/2/_state/state-2.st
data/elasticsearch/nodes/0/indices/customer/2/translog/
data/elasticsearch/nodes/0/indices/customer/2/translog/translog.ckp
data/elasticsearch/nodes/0/indices/customer/2/translog/translog-2.ckp
data/elasticsearch/nodes/0/indices/customer/2/translog/translog-4.tlog
data/elasticsearch/nodes/0/indices/customer/2/translog/translog-2.tlog
data/elasticsearch/nodes/0/indices/customer/2/translog/translog-3.tlog
data/elasticsearch/nodes/0/indices/customer/2/translog/translog-3.ckp
data/elasticsearch/nodes/0/_state/
data/elasticsearch/nodes/0/_state/global-3.st
data/elasticsearch/nodes/0/node.lock
ubuntu@node1:~/docker/elasticsearch$ tar xvf backup/logs.tar 
logs/
ubuntu@node1:~/docker/elasticsearch$ ls -lt
total 20
drwxrwxr-x 5 ubuntu ubuntu 4096 Jun 24 19:52 backup
drwxr-xr-x 3 ubuntu ubuntu 4096 Jun 24 12:08 data
-rw-rw-r-- 1 ubuntu ubuntu 1760 Jun 22 10:25 Dockerfile
drwxr-xr-x 3 ubuntu ubuntu 4096 Jun  9 21:58 config
drwxr-xr-x 2 ubuntu ubuntu 4096 Jun  9 21:58 logs

Run an elasticsearch container


ubuntu@node1:~/docker/elasticsearch/config$ docker run \
--volume=/home/ubuntu/docker/elasticsearch/logs:/usr/share/elasticsearch/logs:rw \
--volume=/home/ubuntu/docker/elasticsearch/data:/usr/share/elasticsearch/data:rw \
--volume=/home/ubuntu/docker/elasticsearch/config:/usr/share/elasticsearch/config:rw \
--restart=always --name esearch -d -p 10.0.2.41:9200:9200 -p 10.0.2.41:9300:9300 \
--net="my-net" --mac-address="7a:07:51:53:31:21" --add-host="esearch:172.28.5.1" --ip="172.28.5.1" elasticsearch:2.3.3
11b8fd312d324364c1169f0c4891e84180ccc7d06361c4764b4bfbd75cb72e23
ubuntu@node1:~/docker/elasticsearch/config$ 
ubuntu@node1:~/docker/elasticsearch/config$ docker ps
CONTAINER ID        IMAGE                 COMMAND                  CREATED             STATUS              PORTS                                                NAMES
11b8fd312d32        elasticsearch:2.3.3   "/docker-entrypoint.s"   10 seconds ago      Up 8 seconds        10.0.2.41:9200->9200/tcp, 10.0.2.41:9300->9300/tcp   esearch
d353dbbe21d3        kibana:4.5.1          "/docker-entrypoint.s"   8 hours ago         Up About an hour    10.0.2.41:5601->5601/tcp                             kibana-es
ubuntu@node1:~/docker/elasticsearch/config$ docker exec -it esearch bash
root@11b8fd312d32:/usr/share/elasticsearch# 
root@11b8fd312d32:/usr/share/elasticsearch# ls -lt
total 44
drwxr-xr-x 3 elasticsearch elasticsearch 4096 Jun 24 05:08 data
drwxr-xr-x 3          1000          1000 4096 Jun  9 14:58 config
drwxr-xr-x 2          1000          1000 4096 Jun  9 14:58 logs
drwxr-xr-x 2 root          root          4096 Jun  9 14:58 bin
drwxr-xr-x 2 root          root          4096 Jun  9 14:58 lib
drwxr-xr-x 5 root          root          4096 Jun  9 14:58 modules
-rw-r--r-- 1 root          root           150 May 17 15:48 NOTICE.txt
-rw-r--r-- 1 root          root          8700 May 17 15:48 README.textile
drwxr-xr-x 2 elasticsearch elasticsearch 4096 May 17 15:48 plugins

Enjoy your elasticseach with saved data after stop and start containers. A data and configuration still on local directory of docker host.



Good luck ;-)

No comments:

Post a Comment