gitlab, build: Win32 distribution jobs for our CI.

The main purpose of these jobs is to only package the strict necessary for a working GIMP under Windows, i.e. getting rid of all unnecessary executables, and inspecting binary dependencies recursively to only package used DLLs. The dll_link.py script is taken from Siril codebase (see commit a86e82a8 on Siril repository, by FlorianBen). This was a very nice idea, and makes for much smaller test archive (Siril is also GPLv3 so licensing is ok for the reuse, also anyway it's just a small independent build script). Moreover having it as a separate job allows to have artifacts with only the finale distribution (artifacts on the build job also have the build directory and the whole prefix, which we want to keep in order to debug when needed). Hopefully I am not missing anything. Siril seems to package more, like various gdk-pixbuf-*.exe, gspawn-*.exe and gdbus.exe. I am wondering if these are actually necessary. I could run GIMP fine without these in quick tests, but I guess I'll have to investigate a bit more to figure this out. That's what nightly builds are for, after all, so hopefully people will report if we miss some runtime dependencies.
2020-09-29 19:38:23 +02:00 · 2020-09-29 19:38:23 +02:00 · 8f8f7e497a
parent b326b68b32
commit 8f8f7e497a
3 changed files with 233 additions and 5 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -237,7 +237,6 @@ gimp-win64:
              -Dgtk-doc=false
              -Dpython=always -Dlua=always
              -Djavascript=always && ninja install &&
-            cp ../build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd $CROSSROAD_PREFIX/bin/ &&
            cp -fr $CROSSROAD_PREFIX/ ../gimp-prefix/
            ' |
      crossroad w64 gimp --run="-"
@ -299,7 +298,6 @@ gimp-win32:
              -Dwmf=disabled  -Dgtk-doc=false
              -Dpython=always -Dlua=always
              -Djavascript=always && ninja install &&
-            cp ../build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd $CROSSROAD_PREFIX/bin/ &&
            cp -fr $CROSSROAD_PREFIX/ ../gimp-prefix/
            ' |
      crossroad w32 gimp --run="-"
@ -345,3 +343,143 @@ sources:
    - mv _build/gimp-*.tar.* .
    - sha256sum gimp-*.tar.* > SHA256SUMS
  needs: ["gimp-distcheck-debian"]
+
+win64-nightly:
+  stage: distribution
+  dependencies:
+    - gimp-win64
+  variables:
+    GIMP_PREFIX: "gimp-prefix"
+    GIMP_DISTRIB: "gimp-w64"
+  artifacts:
+    when: always
+    paths:
+    - gimp-w64
+  script:
+    - apt-get update
+    - apt-get install -y --no-install-recommends python3 binutils-mingw-w64-x86-64
+
+    # Package ressources.
+    - mkdir -p ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/etc ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/include ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/ssl ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/share ${GIMP_DISTRIB}
+
+    # Package executables.
+    - mkdir ${GIMP_DISTRIB}/bin
+    - cp -fr ${GIMP_PREFIX}/bin/gimp*.exe ${GIMP_DISTRIB}/bin/
+    - cp -fr ${GIMP_PREFIX}/bin/glib-compile-schemas.exe ${GIMP_DISTRIB}/bin/
+    - cp -fr ${GIMP_PREFIX}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_DISTRIB}/bin/
+
+    - mkdir ${GIMP_DISTRIB}/libexec
+    - cp -fr ${GIMP_PREFIX}/libexec/gimp*.exe ${GIMP_DISTRIB}/libexec/
+
+    - cp build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd ${GIMP_DISTRIB}/
+
+    # Package library data and modules.
+    - mkdir ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gimp ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gio ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gdk-pixbuf-2.0 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gegl-0.4 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/babl-0.1 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/girepository-1.0 ${GIMP_DISTRIB}/lib/
+
+    # Package needed DLLs only
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-console-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-test-clipboard-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimptool-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_PREFIX}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_PREFIX}/bin/glib-compile-schemas.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+
+    - for dll in ${GIMP_DISTRIB}/lib/babl-0.1/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gegl-0.4/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gio/modules/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gdk-pixbuf-2.0/2.10.0/loaders/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/modules/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/plug-ins/*/*.exe; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+  needs: ["gimp-win64"]
+
+win32-nightly:
+  stage: distribution
+  dependencies:
+    - gimp-win32
+  variables:
+    GIMP_PREFIX: "gimp-prefix"
+    GIMP_DISTRIB: "gimp-w32"
+  artifacts:
+    when: always
+    paths:
+    - gimp-w32
+  script:
+    - apt-get update
+    - apt-get install -y --no-install-recommends python3 binutils-mingw-w64-i686
+
+    # Package ressources.
+    - mkdir -p ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/etc ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/include ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/ssl ${GIMP_DISTRIB}
+    - cp -fr ${GIMP_PREFIX}/share ${GIMP_DISTRIB}
+
+    # Package executables.
+    - mkdir ${GIMP_DISTRIB}/bin
+    - cp -fr ${GIMP_PREFIX}/bin/gimp*.exe ${GIMP_DISTRIB}/bin/
+    - cp -fr ${GIMP_PREFIX}/bin/glib-compile-schemas.exe ${GIMP_DISTRIB}/bin/
+    - cp -fr ${GIMP_PREFIX}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_DISTRIB}/bin/
+
+    - mkdir ${GIMP_DISTRIB}/libexec
+    - cp -fr ${GIMP_PREFIX}/libexec/gimp*.exe ${GIMP_DISTRIB}/libexec/
+
+    - cp build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd ${GIMP_DISTRIB}/
+
+    # Package library data and modules.
+    - mkdir ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gimp ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gio ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gdk-pixbuf-2.0 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/gegl-0.4 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/babl-0.1 ${GIMP_DISTRIB}/lib/
+    - cp -fr ${GIMP_PREFIX}/lib/girepository-1.0 ${GIMP_DISTRIB}/lib/
+
+    # Package needed DLLs only.
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-console-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-test-clipboard-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimptool-2.99.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_PREFIX}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+    - python3 build/windows/crossbuild-gitlab-ci/dll_link.py ${GIMP_PREFIX}/bin/glib-compile-schemas.exe ${GIMP_PREFIX}/ ${GIMP_DISTRIB}
+
+    - for dll in ${GIMP_DISTRIB}/lib/babl-0.1/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gegl-0.4/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gio/modules/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gdk-pixbuf-2.0/2.10.0/loaders/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/modules/*.dll; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+    - for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/plug-ins/*/*.exe; do
+        python3 build/windows/crossbuild-gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${GIMP_DISTRIB};
+      done
+  needs: ["gimp-win32"]
--- a/build/windows/crossbuild-gitlab-ci/dll_link.py
+++ b/build/windows/crossbuild-gitlab-ci/dll_link.py
@ -0,0 +1,90 @@
+#!/usr/bin/python3
+
+################################################################################
+# Small python script to retrieve DLL depencies with objdump
+################################################################################
+
+################################################################################
+# Usage example
+#
+# python3 dll_link.py /path/to/run.exe /winenv/ /path/install
+#
+# In this case, the DLL depencies for executable run.exe will be extracted and
+# copied into /path/install/bin folder. To copy the DLL, the root path to
+# Windows environnement should be passed, here /winenv/.
+
+import sys
+import os
+import subprocess
+import re
+import string
+import shutil
+from pathlib import Path
+
+################################################################################
+# Global variables
+
+# Sets for executable and system DLL
+dll_siril_set = set()
+dll_sys_set = set()
+
+# Install prefix
+prefix = ''
+
+# Windows environement root
+basedir = ''
+
+# Common paths
+binary_dir = '/bin/'
+lib_dir = '/lib/'
+etc_dir = '/etc/'
+share_dir = '/share/'
+
+################################################################################
+# Functions
+
+# Main function
+def main():
+  global basedir
+  global prefix
+
+  if len(sys.argv) < 4:
+    exit(1)
+
+  filename = sys.argv[1]
+  basedir = sys.argv[2]
+  prefix = sys.argv[3]
+
+  recursive(filename)
+  copy_dll(dll_siril_set-dll_sys_set)
+
+# List DLL of an executable file in a recursive way
+def recursive(filename):
+  # Check if DLL exist in /bin folder, if true extract depencies too.
+  if os.path.exists(filename):
+    result = subprocess.run(
+        ['x86_64-w64-mingw32-objdump', '-p', filename], stdout=subprocess.PIPE)
+    out = result.stdout.decode('utf-8')
+    # Parse lines with DLL Name instead of lib*.dll directly
+    items = re.findall(r"DLL Name: \S+.dll", out, re.MULTILINE)
+    for x in items:
+      x = x.split(' ')[2]
+      l = len(dll_siril_set)
+      dll_siril_set.add(x)
+      if len(dll_siril_set) > l:
+        new_dll = basedir + binary_dir + x
+        recursive(new_dll)
+  # Otherwise, it is a system DLL
+  else:
+    dll_sys_set.add(os.path.basename(filename))
+
+# Copy a DLL set into the /prefix/bin directory
+def copy_dll(dll_list):
+  for file in dll_list:
+    full_file_name = os.path.join(basedir + binary_dir, file)
+    if os.path.isfile(full_file_name):
+      shutil.copy(full_file_name, prefix+binary_dir)
+
+
+if __name__ == "__main__":
+  main()
--- a/build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd
+++ b/build/windows/crossbuild-gitlab-ci/gimp-wrapper.cmd
@ -1,3 +1,3 @@
-glib-compile-schemas.exe ..\share\glib-2.0\schemas
-gdk-pixbuf-query-loaders.exe --update-cache
-gimp-2.99.exe
+bin\glib-compile-schemas.exe share\glib-2.0\schemas
+bin\gdk-pixbuf-query-loaders.exe --update-cache
+bin\gimp-2.99.exe