Build linux i686 and windows arm64 wheels

.gitattributes

@@ -0,0 +1,2 @@
+# keep test files byte-identical on Windows checkouts (no CRLF conversion)
+test/testdata/** -text

.github/workflows/wheels.yml

@@ -56,15 +56,16 @@ jobs:
         env:
           # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch
           CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*'
-          # skip PyPy wheels for now, and skip i686 wheels because pytest is failing
-          CIBW_SKIP: pp* *i686
+          # skip PyPy wheels for now
+          CIBW_SKIP: pp*
         run: |
           MATRIX_INCLUDE=$(
             {
               cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
               && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 |  jq -nRc '{"only": inputs, "os": "macos-15-intel"}' \
               && cibuildwheel --print-build-identifiers --platform macos --arch arm64 |  jq -nRc '{"only": inputs, "os": "macos-15"}' \
-              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 |  jq -nRc '{"only": inputs, "os": "windows-latest"}'
+              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 |  jq -nRc '{"only": inputs, "os": "windows-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch ARM64 |  jq -nRc '{"only": inputs, "os": "windows-11-arm"}'
             } | jq -sc
           )
           echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
@@ -89,40 +90,16 @@ jobs:
       if: runner.os == 'Linux'
       uses: docker/setup-qemu-action@v3
 
-    # For Windows, grabbing latest file from MSYS2 is easier than building from source
-    # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file
-    - name: Setup MSYS2 and install file
+    # For Windows, add_libmagic.sh builds libmagic with the MSYS2 toolchain (CIBW_BEFORE_BUILD_WINDOWS below)
+    # ucrt64 over mingw64: libmagic shares the C runtime with python, so from_descriptor works
+    - name: Setup MSYS2
       if: runner.os == 'Windows'
-      uses: msys2/setup-msys2@v2.27.0
+      uses: msys2/setup-msys2@v2.33.0
       with:
-        msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}
-        location: D:\
-        install: >-
-          ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }}
-
-    # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file
-    - name: Copy Windows ddl and mgc
-      if: runner.os == 'Windows'
-      run: |
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic"
-        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic"
-
-    # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file
-    - name: Copy additional 32-bit runtime DLLs
-      if: runner.os == 'Windows' && endsWith(matrix.only, '32')
-      run: |
-        cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic"
-        cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic"
+        msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || endsWith(matrix.only, 'arm64') && 'clangarm64' || 'ucrt64' }}
+        path-type: inherit  # add_libmagic.sh needs the cibuildwheel python
+        install: make
+        pacboy: cc:p libsystre:p ntldd:p
 
     - uses: pypa/cibuildwheel@v2.17.0  # sync version with pip install cibuildwheel above
       timeout-minutes: 10
@@ -133,10 +110,17 @@ jobs:
         # add compiled libmagic to the build directory (to include in the wheel)
         CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh
         CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh
+        CIBW_BEFORE_BUILD_WINDOWS: msys2 -c "bash add_libmagic.sh"
+        # windows: prefer the MSYS2 toolchain over C:\mingw64 on PATH, CFLAGS like https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-file/PKGBUILD
+        # (win32: link the UCRT like python does, as there is no 32-bit ucrt MSYS2 environment)
+        # and skip the x86-only libtool file format check so the tarball's libtool links arm64 DLLs
+        CIBW_ENVIRONMENT_WINDOWS: LC_ALL=en_US.UTF-8 CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" lt_cv_deplibs_check_method=pass_all
         # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag)
         MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }}
-        # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic
-        CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'"
+        # run the test suite against the installed wheel (CIBW_ENVIRONMENT_WINDOWS overrides, so it repeats LC_ALL)
+        CIBW_ENVIRONMENT: LC_ALL=en_US.UTF-8
+        CIBW_TEST_REQUIRES: pytest
+        CIBW_TEST_COMMAND: pytest --import-mode=importlib {project}/test
 
     - uses: actions/upload-artifact@v4
       with:

README.md

@@ -82,9 +82,7 @@ dnf install file-libs
 
 ### Windows
 
-The DLLs that are bundled in the Windows wheels are built by MSYS2 ([mingw32](https://packages.msys2.org/packages/mingw-w64-i686-file) and [mingw64](https://packages.msys2.org/packages/mingw-w64-x86_64-file)).
-
-For ARM64 Windows, you'll need to compile libmagic from source or [install](https://packages.msys2.org/packages/mingw-w64-clang-aarch64-file) from MSYS2.
+The DLLs that are bundled in the Windows wheels are built from source with the MSYS2 toolchain (mingw32, ucrt64 and clangarm64).
 
 ### OSX
 

add_libmagic.sh

@@ -2,17 +2,20 @@
 
 set -euxo pipefail
 
+# on windows (MSYS2), install next to the DLLs libmagic depends on
+prefix="${MINGW_PREFIX:-/usr/local}"
+
 install_source() {
     # install from source
     # https://www.darwinsys.com/file/
-    # https://github.com/file/file/blob/FILE5_46/INSTALL#L51
+    # https://github.com/file/file/blob/FILE5_48/INSTALL#L51
     (
-        version="file-5.46" &&
+        version="file-5.48" &&
         tmpfile="$(mktemp)" &&
         curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" &&
         tar xvf "${tmpfile}" &&
         cd "${version}" &&
-        ./configure &&
+        ./configure --prefix="${prefix}" &&
         make &&
         make install &&
         make installcheck &&
@@ -44,14 +47,14 @@ copy_libmagic() {
     # this python command relies on current working directory containing `./magic/loader.py`
     libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" &&
     cp "${libmagic_path}" "magic" &&
+    # on windows (MSYS2), also copy the DLLs libmagic depends on
+    ( [ -z "${MINGW_PREFIX:-}" ] || ntldd -R "${libmagic_path}" | awk '{print $3}' | cygpath -u -f - | grep "^${MINGW_PREFIX}/bin/" | xargs cp -t "magic" ) &&
     # additionally copy compiled db into magic dir (prefer the one installed by install_source)
-    ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) &&
+    ( ( cp "${prefix}/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) &&
     # check what was copied
     ls -ltra magic
 }
 
-# skip windows (taken care of separately in wheels.yml)
-python -c 'import platform; assert platform.system() != "Windows"' || ( echo "skipping on windows" && exit 0 )
 # prefer a recent build from source
 install_source || install_precompiled
 # files to be copied into the wheel

test/python_magic_test.py

@@ -93,8 +93,8 @@ class TestFile:
         (NO_SOFT, ["data"]),
     ],
     b"test.snappy.parquet": [
-        (COMMON_MIME, ["application/octet-stream"]),
-        (COMMON_PLAIN, ["data", "Apache Parquet", "Par archive data"]),
+        (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]),
+        (COMMON_PLAIN, ["data", "Apache Parquet", "Apache Parquet file", "Par archive data"]),
         (NO_SOFT, ["data"]),
     ],
     b"test.json": [
@@ -210,6 +210,9 @@ def test_all_cases(self):
         os.environ["TZ"] = "UTC"
         try:
             for filename, cases in CASES.items():
+                if sys.platform == "win32" and filename == b"\xce\xbb":
+                    # libmagic opens paths with the narrow (ANSI codepage) api on Windows
+                    continue
                 filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename)
                 print("test case ", filename, file=sys.stderr)
                 for flag_variants, outputs in cases:
@@ -282,6 +285,11 @@ def test_getparam(self):
         except NotImplementedError:
             pass
 
+    def test_from_buffer_hls_mp4(self):
+        # HLS MP4 video, only recognised in recent versions of libmagic (and broken in 5.47 ref #362)
+        buf = b"\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00"
+        self.assertEqual("video/mp4", magic.from_buffer(buf, mime=True))
+
     def test_name_count(self):
         m = magic.Magic()
         with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f:
@@ -296,6 +304,7 @@ def test_pathlike(self):
         m = magic.Magic(mime=True)
         self.assertEqual("application/pdf", m.from_file(path))
 
+    @unittest.skipIf(sys.platform == "win32", "libmagic has no symlink support on Windows")
     def test_symlink(self):
         # TODO: 3.0
         if not hasattr(tempfile, "TemporaryDirectory"):