diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..62cb0ac --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# keep test files byte-identical on Windows checkouts (no CRLF conversion) +test/testdata/** -text diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 867d2e9..e816691 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -56,15 +56,16 @@ jobs: env: # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*' - # skip PyPy wheels for now, and skip i686 wheels because pytest is failing - CIBW_SKIP: pp* *i686 + # skip PyPy wheels for now + CIBW_SKIP: pp* run: | MATRIX_INCLUDE=$( { cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -nRc '{"only": inputs, "os": "macos-15-intel"}' \ && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -nRc '{"only": inputs, "os": "macos-15"}' \ - && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch ARM64 | jq -nRc '{"only": inputs, "os": "windows-11-arm"}' } | jq -sc ) echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT @@ -89,40 +90,16 @@ jobs: if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 - # For Windows, grabbing latest file from MSYS2 is easier than building from source - # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file - - name: Setup MSYS2 and install file + # For Windows, add_libmagic.sh builds libmagic with the MSYS2 toolchain (CIBW_BEFORE_BUILD_WINDOWS below) + # ucrt64 over mingw64: libmagic shares the C runtime with python, so from_descriptor works + - name: Setup MSYS2 if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2.27.0 + uses: msys2/setup-msys2@v2.33.0 with: - msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }} - location: D:\ - install: >- - ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }} - - # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file - - name: Copy Windows ddl and mgc - if: runner.os == 'Windows' - run: | - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic" - - # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file - - name: Copy additional 32-bit runtime DLLs - if: runner.os == 'Windows' && endsWith(matrix.only, '32') - run: | - cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic" - cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic" + msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || endsWith(matrix.only, 'arm64') && 'clangarm64' || 'ucrt64' }} + path-type: inherit # add_libmagic.sh needs the cibuildwheel python + install: make + pacboy: cc:p libsystre:p ntldd:p - uses: pypa/cibuildwheel@v2.17.0 # sync version with pip install cibuildwheel above timeout-minutes: 10 @@ -133,10 +110,17 @@ jobs: # add compiled libmagic to the build directory (to include in the wheel) CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh + CIBW_BEFORE_BUILD_WINDOWS: msys2 -c "bash add_libmagic.sh" + # windows: prefer the MSYS2 toolchain over C:\mingw64 on PATH, CFLAGS like https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-file/PKGBUILD + # (win32: link the UCRT like python does, as there is no 32-bit ucrt MSYS2 environment) + # and skip the x86-only libtool file format check so the tarball's libtool links arm64 DLLs + CIBW_ENVIRONMENT_WINDOWS: LC_ALL=en_US.UTF-8 CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" lt_cv_deplibs_check_method=pass_all # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} - # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic - CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'" + # run the test suite against the installed wheel (CIBW_ENVIRONMENT_WINDOWS overrides, so it repeats LC_ALL) + CIBW_ENVIRONMENT: LC_ALL=en_US.UTF-8 + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest --import-mode=importlib {project}/test - uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index deb06fe..3512187 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,7 @@ dnf install file-libs ### Windows -The DLLs that are bundled in the Windows wheels are built by MSYS2 ([mingw32](https://packages.msys2.org/packages/mingw-w64-i686-file) and [mingw64](https://packages.msys2.org/packages/mingw-w64-x86_64-file)). - -For ARM64 Windows, you'll need to compile libmagic from source or [install](https://packages.msys2.org/packages/mingw-w64-clang-aarch64-file) from MSYS2. +The DLLs that are bundled in the Windows wheels are built from source with the MSYS2 toolchain (mingw32, ucrt64 and clangarm64). ### OSX diff --git a/add_libmagic.sh b/add_libmagic.sh index b1133ff..cea1a84 100755 --- a/add_libmagic.sh +++ b/add_libmagic.sh @@ -2,17 +2,20 @@ set -euxo pipefail +# on windows (MSYS2), install next to the DLLs libmagic depends on +prefix="${MINGW_PREFIX:-/usr/local}" + install_source() { # install from source # https://www.darwinsys.com/file/ - # https://github.com/file/file/blob/FILE5_46/INSTALL#L51 + # https://github.com/file/file/blob/FILE5_48/INSTALL#L51 ( - version="file-5.46" && + version="file-5.48" && tmpfile="$(mktemp)" && curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" && tar xvf "${tmpfile}" && cd "${version}" && - ./configure && + ./configure --prefix="${prefix}" && make && make install && make installcheck && @@ -44,14 +47,14 @@ copy_libmagic() { # this python command relies on current working directory containing `./magic/loader.py` libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" && cp "${libmagic_path}" "magic" && + # on windows (MSYS2), also copy the DLLs libmagic depends on + ( [ -z "${MINGW_PREFIX:-}" ] || ntldd -R "${libmagic_path}" | awk '{print $3}' | cygpath -u -f - | grep "^${MINGW_PREFIX}/bin/" | xargs cp -t "magic" ) && # additionally copy compiled db into magic dir (prefer the one installed by install_source) - ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) && + ( ( cp "${prefix}/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) && # check what was copied ls -ltra magic } -# skip windows (taken care of separately in wheels.yml) -python -c 'import platform; assert platform.system() != "Windows"' || ( echo "skipping on windows" && exit 0 ) # prefer a recent build from source install_source || install_precompiled # files to be copied into the wheel diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 651c0be..9b6db8a 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -93,8 +93,8 @@ class TestFile: (NO_SOFT, ["data"]), ], b"test.snappy.parquet": [ - (COMMON_MIME, ["application/octet-stream"]), - (COMMON_PLAIN, ["data", "Apache Parquet", "Par archive data"]), + (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]), + (COMMON_PLAIN, ["data", "Apache Parquet", "Apache Parquet file", "Par archive data"]), (NO_SOFT, ["data"]), ], b"test.json": [ @@ -210,6 +210,9 @@ def test_all_cases(self): os.environ["TZ"] = "UTC" try: for filename, cases in CASES.items(): + if sys.platform == "win32" and filename == b"\xce\xbb": + # libmagic opens paths with the narrow (ANSI codepage) api on Windows + continue filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) print("test case ", filename, file=sys.stderr) for flag_variants, outputs in cases: @@ -282,6 +285,11 @@ def test_getparam(self): except NotImplementedError: pass + def test_from_buffer_hls_mp4(self): + # HLS MP4 video, only recognised in recent versions of libmagic (and broken in 5.47 ref #362) + buf = b"\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00" + self.assertEqual("video/mp4", magic.from_buffer(buf, mime=True)) + def test_name_count(self): m = magic.Magic() with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f: @@ -296,6 +304,7 @@ def test_pathlike(self): m = magic.Magic(mime=True) self.assertEqual("application/pdf", m.from_file(path)) + @unittest.skipIf(sys.platform == "win32", "libmagic has no symlink support on Windows") def test_symlink(self): # TODO: 3.0 if not hasattr(tempfile, "TemporaryDirectory"):