Add migrate test from existing volumes (previous OpenWrt installations)

test/docker-vackup/LICENSE.txt

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

test/docker-vackup/README.md

@@ -0,0 +1,62 @@
+# Vackup: Backup and Restore Docker Volumes
+
+[![Lint Code Base](https://github.com/BretFisher/docker-vackup/actions/workflows/linter.yml/badge.svg)](https://github.com/BretFisher/docker-vackup/actions/workflows/linter.yml)
+
+**This is now an [Official Docker Desktop Extension called "Volumes Backup & Share"](https://hub.docker.com/extensions/docker/volumes-backup-extension) which has more features, but I'll keep this repo around for historial purposes.**
+
+Vackup: (contraction of "volume backup")
+
+Easily backup and restore Docker volumes using either tarballs or container images.
+It's designed for running from any host/container where you have the docker CLI.
+
+Note that for open files like databases,
+it's usually better to use their preferred backup tool to create a backup file,
+but if you stored that file on a Docker volume,
+this could still be a way you get the Docker volume into a image or tarball
+for moving to remote storage for safe keeping.
+
+`export`/`import` commands copy files between a local tarball and a volume.
+For making volume backups and restores.
+
+`save`/`load` commands copy files between an image and a volume.
+For when you want to use image registries as a way to push/pull volume data.
+
+Usage:
+
+`vackup export VOLUME FILE`
+  Creates a gzip'ed tarball in current directory from a volume
+
+`vackup import FILE VOLUME`
+  Extracts a gzip'ed tarball into a volume
+
+`vackup save VOLUME IMAGE`
+  Copies the volume contents to a busybox image in the /volume-data directory
+
+`vackup load IMAGE VOLUME`
+  Copies /volume-data contents from an image to a volume
+
+## Install
+
+Download the `vackup` file in this repository to your local machine in your shell path and make it executable.
+
+```shell
+sudo curl -sSL https://raw.githubusercontent.com/BretFisher/docker-vackup/main/vackup -o /usr/local/bin/vackup && sudo chmod +x /usr/local/bin/vackup
+```
+
+## Error conditions
+
+If any of the commands fail, the script will check to see if a `VACKUP_FAILURE_SCRIPT`
+environment variable is set.  If so it will run it and pass the line number the error
+happened on and the exit code from the failed command.  Eg,
+
+```shell
+# /opt/bin/vackup-failed.sh
+LINE_NUMBER=$1
+EXIT_CODE=$2
+send_slack_webhook "Vackup failed on line number ${LINE_NUMBER} with exit code ${EXIT_CODE}!"
+```
+
+```shell
+export VACKUP_FAILURE_SCRIPT=/opt/bin/vackup-failed.sh
+./vackup export ......
+```

test/docker-vackup/vackup

@@ -0,0 +1,189 @@
+#!/bin/bash
+# Docker Volume File Backup and Restore Tool
+# Easily tar up a volume on a local (or remote) engine
+# Inspired by CLIP from Lukasz Lach
+
+set -Eeo pipefail
+
+handle_error() {
+  exit_code=$?
+  if [ -n "${VACKUP_FAILURE_SCRIPT}" ]; then
+    /bin/bash "${VACKUP_FAILURE_SCRIPT}" "$1" $exit_code
+  fi
+  exit $exit_code
+}
+
+trap 'handle_error $LINENO' ERR
+
+usage() {
+cat <<EOF
+
+"Docker Volume Backup". Replicates image management commands for volumes.
+
+export/import copies files between a host tarball and a volume. For making
+  volume backups and restores.
+
+save/load copies files between an image and a volume. For when you want to use 
+  image registries as a way to push/pull volume data.
+
+Usage: 
+
+vackup export VOLUME FILE
+  Creates a gzip'ed tarball in current directory from a volume
+
+vackup import FILE VOLUME
+  Extracts a gzip'ed tarball into a volume
+
+vackup save VOLUME IMAGE
+  Copies the volume contents to a busybox image in the /volume-data directory
+
+vackup load IMAGE VOLUME
+  Copies /volume-data contents from an image to a volume
+
+EOF
+}
+
+if [ -z "$1" ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
+    usage
+    exit 0
+fi
+
+cmd_export() {
+    VOLUME_NAME="$2"
+    FILE_NAME="$3"
+    
+    if [ -z "$VOLUME_NAME" ] || [ -z "$FILE_NAME" ]; then
+        echo "Error: Not enough arguments"
+        usage
+        exit 1
+    fi
+    
+    if ! docker volume inspect --format '{{.Name}}' "$VOLUME_NAME";
+    then
+        echo "Error: Volume $VOLUME_NAME does not exist"
+        exit 1
+    fi
+
+# TODO: check if file exists on host, if it does
+# create a option for overwrite and check if that's set
+# TODO: if FILE_NAME starts with / we need to error out
+# unless we can translate full file paths
+
+    if ! docker run --rm \
+      -v "$VOLUME_NAME":/vackup-volume \
+      -v "$(pwd)":/vackup \
+      busybox \
+      tar -zcvf /vackup/"$FILE_NAME" /vackup-volume;
+    then
+        echo "Error: Failed to start busybox backup container"
+        exit 1
+    fi
+
+    echo "Successfully tar'ed volume $VOLUME_NAME into file $FILE_NAME"
+}
+
+cmd_import() {
+    FILE_NAME="$2"
+    VOLUME_NAME="$3"
+    
+    if [ -z "$VOLUME_NAME" ] || [ -z "$FILE_NAME" ]; then
+        echo "Error: Not enough arguments"
+        usage
+        exit 1
+    fi
+    
+    if ! docker volume inspect --format '{{.Name}}' "$VOLUME_NAME";
+    then
+        echo "Error: Volume $VOLUME_NAME does not exist"
+        docker volume create "$VOLUME_NAME"
+    fi
+
+# TODO: check if file exists on host, if it does
+# create a option for overwrite and check if that's set
+# TODO: if FILE_NAME starts with / we need to error out
+# unless we can translate full file paths    
+
+    if ! docker run --rm \
+      -v "$VOLUME_NAME":/vackup-volume \
+      -v "$(pwd)":/vackup \
+      busybox \
+      tar -xvzf /vackup/"$FILE_NAME" -C /; 
+    then
+        echo "Error: Failed to start busybox container"
+        exit 1
+    fi
+
+    echo "Successfully unpacked $FILE_NAME into volume $VOLUME_NAME"
+}
+
+cmd_save() {
+    VOLUME_NAME="$2"
+    IMAGE_NAME="$3"
+
+    if [ -z "$VOLUME_NAME" ] || [ -z "$IMAGE_NAME" ]; then
+        echo "Error: Not enough arguments"
+        usage
+        exit 1
+    fi
+
+    if ! docker volume inspect --format '{{.Name}}' "$VOLUME_NAME"; 
+    then
+        echo "Error: Volume $VOLUME_NAME does not exist"
+        exit 1
+    fi
+
+    if ! docker run \
+      -v "$VOLUME_NAME":/mount-volume \
+      busybox \
+      cp -Rp /mount-volume/. /volume-data/;
+    then
+        echo "Error: Failed to start busybox container"
+        exit 1
+    fi
+
+    CONTAINER_ID=$(docker ps -lq)
+
+    docker commit -m "saving volume $VOLUME_NAME to /volume-data" "$CONTAINER_ID" "$IMAGE_NAME"
+
+    docker container rm "$CONTAINER_ID"
+  
+    echo "Successfully copied volume $VOLUME_NAME into image $IMAGE_NAME, under /volume-data"
+}
+
+cmd_load() {
+    IMAGE_NAME="$2"
+    VOLUME_NAME="$3"
+    
+    if [ -z "$VOLUME_NAME" ] || [ -z "$IMAGE_NAME" ]; then
+        echo "Error: Not enough arguments"
+        usage
+        exit 1
+    fi
+
+    if ! docker volume inspect --format '{{.Name}}' "$VOLUME_NAME"; 
+    then
+      echo "Volume $VOLUME_NAME does not exist, creating..."
+      docker volume create "$VOLUME_NAME"
+    fi
+    
+    if ! docker run --rm \
+      -v "$VOLUME_NAME":/mount-volume \
+      "$IMAGE_NAME" \
+      cp -Rp /volume-data/. /mount-volume/; 
+    then
+        echo "Error: Failed to start container from $IMAGE_NAME"
+        exit 1
+    fi
+
+    echo "Successfully copied /volume-data from $IMAGE_NAME into volume $VOLUME_NAME"
+}
+
+COMMAND="$1"
+case "$COMMAND" in
+  export) cmd_export "$@" ;;
+  import) cmd_import "$@" ;;
+  save) cmd_save "$@" ;;
+  load) cmd_load "$@" ;;
+esac
+
+exit 0

test/tests/openwrt_docker_test.py

@@ -19,7 +19,7 @@ def parameter(request):
 
 # Adapt docker-compose.yml to our test needs
 @pytest.fixture(scope="session")
-def docker_compose_file(pytestconfig, parameter):
+def docker_compose_file(pytestconfig, parameter, docker_compose_project_name):
         
     compose_file_template = os.path.join(str(pytestconfig.rootdir), "tests", "docker-compose.yml.template")
     compose_file = os.path.join(str(pytestconfig.rootdir), "tests", "docker-compose.yml.generated")
@@ -28,12 +28,23 @@ def docker_compose_file(pytestconfig, parameter):
     with open(compose_file_template) as f:
         list_doc = yaml.safe_load(f)
 
-    # Change enviroment variable
     if parameter:
-        if len(parameter) == 2:
+        # Change environment variables
+        if len(parameter) >= 2:
             env_var = parameter[0]
             value = parameter[1]
-            list_doc['services']['openwrt']['environment'][env_var] = value
+            if env_var != "": 
+                list_doc['services']['openwrt']['environment'][env_var] = value
+
+        # Load volume data
+        if len(parameter) == 3:
+            # We don't care about the first two parameters, see above
+            # Let's use only the third one
+            data_volume_backup = parameter[2]
+            vackup_dir = os.path.join(str(pytestconfig.rootdir), "docker-vackup")
+            vackup = os.path.join(vackup_dir, "vackup") # Funny name ;-)
+            
+            os.system(f"cd {vackup_dir}/../test_data/ && {vackup} import {data_volume_backup} {docker_compose_project_name}_data >/dev/null 2>/dev/null")
 
     # Save docker-compose file
     with open(compose_file, "w") as f:
@@ -183,6 +194,16 @@ def is_openwrt_booted():
     else:
         return False
 
+def get_logs():
+    process = subprocess.run(['docker','logs','openwrt'], 
+        stdout=subprocess.PIPE, 
+        stderr=subprocess.PIPE,
+        universal_newlines=True)
+    
+    return process.stdout + process.stderr
+
+def is_specific_log(log_text):
+    return log_text in get_logs()
 
 # ************************ Tests ************************
 
@@ -292,7 +313,7 @@ def test_openwrt_wan(docker_services, parameter):
             return
         
         case _: # Usage of real Ethernet interface e.g. 'eth0'
-            # This test is most likely only working in a github action enviroment because multiple VM are necessary to test it. See the action file, please
+            # This test is most likely only working in a github action environment because multiple VM are necessary to test it. See the action file, please
             # Try to get IP address from WAN-VM
             response = run_openwrt_shell_command("udhcpc", "-i", "eth1")
             assert ('udhcpc: setting default routers: 192.168.22.1' in response['out-data']) == True
@@ -333,6 +354,7 @@ def test_cpu_num(docker_services, parameter):
         
     assert parameter[1] == cpu_num
 
+
 def test_additional_installed_packages(docker_services):
 
     # Get list of packages to be installed in OpenWrt
@@ -363,3 +385,44 @@ def test_additional_installed_packages(docker_services):
     # Check if all additional OpenWrt packages in Docker image are installed in OpenWrt
     for package in package_list_docker:
         assert any(package in s for s in package_list_openwrt_full) == True
+
+
+@pytest.mark.parametrize("parameter", 
+    [('','','20240922_test_volume_openwrt_23.05.4.tar.gz')], indirect=True,
+    ids=['20240922_test_volume_openwrt_23.05.4'])
+def test_openwrt_migrate_existing_volume(docker_services):
+    docker_services.wait_until_responsive(
+        timeout=90.0, pause=1, check=lambda: is_specific_log('Booting image using QEMU emulator')
+    )
+
+    assert ('upgrade: Saving config files' in get_logs()) == True
+    assert ('upgrade: Restoring config files' in get_logs()) == True
+
+
+@pytest.mark.parametrize("parameter", 
+    [('','','20240922_test_volume_openwrt_23.05.4.tar.gz')], indirect=True,
+    ids=['20240922_test_volume_openwrt_23.05.4'])
+def test_openwrt_migrate_settings(docker_services):
+    docker_services.wait_until_responsive(
+        timeout=90.0, pause=1, check=lambda: is_openwrt_booted()
+    )
+
+    # Here we test a custom configuration that is stored in the existing volume.
+    # Currently hostname and an additional interface are configured.
+    
+    # Check for host name 'TestRouter'
+    response = run_openwrt_shell_command("uci", "get", "system.@system[0].hostname")
+    assert ('TestRouter' in response['out-data']) == True
+
+    # Check for test LAN
+    response = run_openwrt_shell_command("uci", "show", "network.TestLan")
+    assert ("network.TestLan.ipaddr='192.168.40.1'" in response['out-data']) == True
+
+
+def test_kvm(docker_services):
+    docker_services.wait_until_responsive(
+        timeout=90.0, pause=1, check=lambda: is_specific_log('Checking for KVM')
+    )
+
+    time.sleep(1)
+    assert ('KVM detected' in get_logs()) == True