diff --git a/poetry.lock b/poetry.lock index 2d6d7f8..adada52 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "anyascii" @@ -49,17 +49,22 @@ lxml = ["lxml"] [[package]] name = "bibtexparser" -version = "1.4.4" +version = "2.1.0" description = "Bibtex parser for python 3" optional = false -python-versions = "*" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "bibtexparser-1.4.4.tar.gz", hash = "sha256:093b6c824f7a71d3a748867c4057b71f77c55b8dbc07efc993b781771520d8fb"}, + {file = "bibtexparser-2.1.0-py3-none-any.whl", hash = "sha256:3b4b1de50ce45ee9091ccca72e1efc8732ab3ef615b70bcecc537c17cd84753e"}, + {file = "bibtexparser-2.1.0.tar.gz", hash = "sha256:9f48851eef4693cb7ef6897f7628966c5d0ef49b6403b13a862679a4c313d176"}, ] [package.dependencies] -pyparsing = ">=2.0.3" +pylatexenc = ">=2.10,<3.0" + +[package.extras] +docs = ["sphinx"] +test = ["jupyter", "pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "bs4" @@ -1202,7 +1207,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -1596,6 +1601,18 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pylatexenc" +version = "2.11" +description = "Simple LaTeX parser providing latex-to-unicode and unicode-to-latex conversion" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pylatexenc-2.11-py2.py3-none-any.whl", hash = "sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0"}, + {file = "pylatexenc-2.11.tar.gz", hash = "sha256:305a072a99ce736246049c9da05841b9d718c0f7ea8888f5f596cf15cb621053"}, +] + [[package]] name = "pyoxigraph" version = "0.3.22" @@ -1790,6 +1807,122 @@ files = [ {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] +[[package]] +name = "rapidfuzz" +version = "3.14.6" +description = "rapid fuzzy string matching" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "rapidfuzz-3.14.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c0dd0d765184366b6e213a8af3b0b3bb39dad27943bbfb193515d4ff96ac82a"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c61cade182f130c9903231946bd1074539121721693a918e7b70382ae802bd8"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3781cf14f9fc933d7198c2b25a8bbbd1a62b752746d5cd26de14957edc0e802f"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71a5bbfd00da1963f27dd1432068929694cf0e00007ae2b9c1ad2a187ec29a16"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eabaf06ca4896c59cfd9162480f0d37a15a2304ce2efe83ae2bbcfa1cf13534e"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d5d90bae3c6fb7ea34da968c9f23070e8440edb827a28b242580e0108110b14"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d6b58daadbe6974884ec39aee30cfb8bd2e126f8d03503f0069f70d5e84656a3"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ab4386ef7c2cb3e5eb46e815be49715dfcd301bb9f0a431f18da7aa0007de54f"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:33a2f7faedaa3608c4876c41b448fc786d54e6cd7c6e732f7de466319b5a73c2"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:adb160a100f6122aa45c78d686e198da3f9e815d4182e0c4fe730608479f7f9c"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ad60297c001d15af24338440bca85dfee8710e9e3222733c906b33e89d986166"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d5b1cfa67bbe6239a643bca1d986f8a07e0a045286c674946e1648c132baa46"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-win32.whl", hash = "sha256:46ddb42af4cad3ac9d5e0c97ee1e687500c529a1ad5cbf9c949ce35f6edd4537"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-win_amd64.whl", hash = "sha256:737a57cbca3e5c16decac86e205727bcd4b99c52f77c48bb44123078c5cd9a7a"}, + {file = "rapidfuzz-3.14.6-cp311-cp311-win_arm64.whl", hash = "sha256:19c1cda8198cc57ffd4ff69a1c02cbe4297e9ca7b506bca03ec584da0a9fe1ff"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b46cecf27025e7a934332ade033e6a394da8a493f19fa1d835e3b2968a4ff7da"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1901414b135afb1a7f4b1ef940b95523b49cc5642aecf02af740f37567e98137"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96a548979cd939b2c69358a0f5088a408524fbf7454f04bf90939fa971e64310"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b22ef7e5e2341efc6216b666491022027b984e5aef93446064742f43f3c1d926"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f0d2d95c787d812b9106cfbcb94ad37a49f59df9287e00a75eb61afc246e8759"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0debb5f43662ea84d2f0228a0c7407ff647f9c3d13f3b692efff0cde46eebce0"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:1d253e1fe44648242a0029b42ba23adf238ed2a7eb3d8ed0a03731a23f074ae0"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e06c6050c9bf6cd72305e3e6a293918b2b92cf2a067007585a53898624902e3c"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d85a6e9180e53cde95c95dfeb05a2ac94ead4d9d803a8fd186d2719a678b8483"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:35db2670f69fa3a4eb4741055581477ff92f2cf39e7e06f43ebcb97c2192fe7c"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f9d93e5424d1e4c103b57906b8beba270e680afda3ffdff7ea3bc6173b37083c"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9b0a501f37fb852c54469375baa25874246b3bbc8b6e21fb4cd186a32335868"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-win32.whl", hash = "sha256:9e974251a9833791bc557b46f975676a56c2d58946f795cd2964b095496dfdcc"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-win_amd64.whl", hash = "sha256:cfca36e4612208875e08611a779164b6cb8900ab8bbd3d82d4cfdfae9efbfac9"}, + {file = "rapidfuzz-3.14.6-cp312-cp312-win_arm64.whl", hash = "sha256:96bbd5a1c67d135334d02fae74f1d933fdda204ea03d544a59dab6b1cbfbf565"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:55dc9a55924b4ecfcf4a60a701bcfae7d9daf0129c41dc16139270d75be0996c"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bba0e9fad4dbea80227cde9cef3aaa984a934a84aec5f7505532e19838b14769"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b34b7ee4f4f760690d6477163aabbec05705b5dd764cb6c3a6ba95aa1fffc42"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:abe92a70134c8b40790bb5c78b2a0a790686c26e83b6e99a456127ca141fe06a"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:659b41570fcc6e02631ac361c47cc8db9ad26d740e4be2177df1b63005a49174"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bb896f89a387219c671ebc33c4a636b222010cc3c5c83884a7fc8707bf0bbf9"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:11d76bb2b2cd038df708ae18f521fb3a50af477cc5a0dffce812da43a2f1beb3"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:28e9ce91bd41a8203185887ef9b1541a891aa61c5c1cb2e46f1689cd4288d372"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:864658e5a10d249a2277374e800f944fe990346d70eea6f3a51b712b6dd01984"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:3c2444f5cd757ded2c3ba8b1734253b801b9b2ba9ecb3ee40cd505cebbfa7341"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2cc9b5dde0ac89f7856f997ef917cac8e18e9dea473e9b3090a84bd600de6a91"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:faebff9b9a287fb673f9a66465a7e03043601c9bfe5e71c3f91b3f2e7b8a37f6"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-win32.whl", hash = "sha256:4406b2517b85febcf9419f8fbcdfbd534872ea32608050f9562224933ca49a4c"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-win_amd64.whl", hash = "sha256:c69fb0e064d10c79908dcda76d7ca8ecdf8393a39acbb74dbad3f709f2c60e95"}, + {file = "rapidfuzz-3.14.6-cp313-cp313-win_arm64.whl", hash = "sha256:a0c8bef04f6b1d9fdbb319576350af53151a64692d477db7d4844c220bc8e212"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f8d6718e7edacdb16455c0472e7552fd518decb91e91250c58784fd6163f54f"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8fa7d45388dec34a86038f2a38380f4922b74b5dd8991247f629a531178db10f"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:760ee152af5e8b4d241a469f933ba2d7215248618ae19770fec7d80d9e149db6"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dbe3378db3ae0453accf6196e2ed943f43d416cfacdcb8883db105bc14a0130f"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ddb0ddf3ee616fdc066add4ef05639c5cf59b58d83779b6023488e5435f6191"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:08bc63b88048376114d1e66cf8fa6926495d03bb873eb87854fa74cf6848a70b"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:50cd6718bcda7ec5293635a9d0b3fb5906251013d3b99ca403ba9dfa8965f661"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:63b0e84faec3c5706cae8ae51246ff103407d54efa32a615a548b7b67392ebcf"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9080a730fdcf3cb8a07464c90f9cf40c1b4ffc73a8375b56a8898aba619dda30"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:178557c7a50c8c8d65369ede7f3d845bf23590a951c9a368caf166b105d58cf3"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:44f1cddbc2010700e2d88063d0ab64183efe2578d9b52770ce1cd283dda230c5"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:17081a0e904c12bb4ed49619a2bbb6528f6af00fe850e7ace22487bfd2aea455"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-win32.whl", hash = "sha256:9e00c8c9500aacbc0c52b66369f54533ecbdcb92e5aa87e160fc8e293000a696"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-win_amd64.whl", hash = "sha256:41ee893c4d7d0fb1844f6cad966540a833784b3bad2c239a0d80195d9231cef4"}, + {file = "rapidfuzz-3.14.6-cp314-cp314-win_arm64.whl", hash = "sha256:10576c39fe6a49fad0bf1069371a77300ce166a3f36d2900d2d0bae08f297104"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1b0a9546a7328d3cfc2f1385501db7c4c374fb566dc1a3b22ad56092846c0134"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9989280902b9c4ecf7de95fbb906e94df0d8c047290ed315c7aa1760cec9b3de"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc166efa4ca2fc9cc52e43784a54cbea95fc0e03e533f8266ef66b1c04c7cb76"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:32352a3ed1aad9c097d31fd4f2eece3030169e2de3dedde7a2fadc2652b768ad"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ecb45d616002751b58914d5b7c2e66acd39e12242be12717a1258148a1b36526"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f9ad513e3a3e045b60b421d5cd3887ae0a33b38fc6c6db3ea5e27c0a2e0412c"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:f35723caef8cc31b6f34209708fb172fc88bab0077c12e9b36bbb829baaf1b16"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:408b2e8e8c1ac71b57f0923cf964d6932539725e07b69e70ec66f22c4a403891"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:5667c56fdc902fa1e12449b5c042e8b1c7e9b30040db20c396fbdb3d0a750866"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:76a122fc573df603deb5fb827df31bb5efbd0826b50bb7aeca8535a6e8c70cf9"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:e221366e24709b9d41d5f9cc99053b04cfc575d429e956a82cfbc4c4e9e8860a"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:36710ff214b7a8049d26a9c81d99948026593cacb47663742c4119072b651ecd"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-win32.whl", hash = "sha256:66ece6f5e2586c742fc3e0b8487e06783d27c6c24adcdcfdd7f306afbd8b5737"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cab4a932cec02d09471e2c9f1434049ef5bfe1f6e646ff10939c222dc610ad60"}, + {file = "rapidfuzz-3.14.6-cp314-cp314t-win_arm64.whl", hash = "sha256:b056ce19eaea2ea70c6a6fb387a605ca2af8979de5b9d507597e8012820ddb14"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bc3d74d18543ddfbc8babe1faadb19927a7999fd0d01181cce9e721c14c36ab6"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:aaa83b633d877a05d549d2073629134998d1b3b9dbc114873d3ff4277984979f"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbe6a62f71fcbca72acbf5a30e53380600369f257f951d664d81d30c0c598595"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b82c21c30568e096ef2a9dda7d45c379e6141694e0472dac73bc4372ce13ccee"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:fc950bb77105a2717d03d9f9c9e21e9ace7df2b8e864dd91edef7e32fa143be2"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c53a269bdbd71ffbc856d3db9e609478251001ee272507578fa838bc2bd421fe"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:bf4fb0f19c9dfce7a908c3e309753602ce3edb83bb74e9ff997e278765bf89df"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:189ce2bf14938bfa003fbbe7e6da7584ed6ebbc4c560686255dbc20e2829f470"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-win32.whl", hash = "sha256:7ca0f498bf771a87557e6d8b573aa6cf3daded58ae2eaeb6973618ce3e1615ad"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-win_amd64.whl", hash = "sha256:d4c5adb921b67dd79ffc0a14f92b9f8df3d012e66aab340b154ed87014229d93"}, + {file = "rapidfuzz-3.14.6-cp315-cp315-win_arm64.whl", hash = "sha256:c9d135fb93709d707577da8a7a8ffc7283525a5b6d0ce55aa3724be5639ed65b"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:dd89abd1c4b3776c3471a817216830bd275441c8344bbda5d51a3bffe1e0fbdf"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:eab2d4680d7f438dbb1d484b187d59a943edea9c83f792c764a0c148a417a60a"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8683fefdd3484d64a191b3efbc8cbe9162c3eac891fd62d0a1b70e117ffcd434"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2bc7af3a699371a941aac86dc8a79ac92adeb3c2add2aab02230e76068a0029e"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:40c2753e2d4dc96b25f8a25adc23ab0bb6cfd8bc8125a1753ac4b037d6ff6511"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:36a37ddc729c33618d89fa221d3333b9b956dc38cf15d31301e6169d962399a3"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:635f242f4bdf05d1477fa409815bd73e5f78896773ace84997bc472ffeef685f"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:40d0cd9c82083aeb30bae8dee265ae571e6748d0d7b222ddd777f33d95a3b712"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-win32.whl", hash = "sha256:15da2b258908eb38853c1a6a58a1d09d9aad9c721e03a68c8ba691cd31dff739"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-win_amd64.whl", hash = "sha256:3d502769263318690d4f6638b08483979d1b88cdc7c6f087482eea935fde4031"}, + {file = "rapidfuzz-3.14.6-cp315-cp315t-win_arm64.whl", hash = "sha256:07c7aa0b1e4b9999a54f9e73317d6743ff85442c8ef7b7fbbe6b190fd37d9e75"}, + {file = "rapidfuzz-3.14.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0844066900cdc9909ce4ab4fb5ba1d8e0c021252d770f2ea476f3443df1d22ef"}, + {file = "rapidfuzz-3.14.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1398bd2c197b79bfc40b615999fd3599dc60265fdd5b59edc18156ae048c4cde"}, + {file = "rapidfuzz-3.14.6-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2fc748d1fde4109e5d0dab27f1e61f53b3136a235dfee5a4fb579da44808b6a"}, + {file = "rapidfuzz-3.14.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b42536675c930cb76b7998bfc4d8e59cb35d8df47f2103020265743b6b2ccd2a"}, + {file = "rapidfuzz-3.14.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1e6911e3a14971719ddc35af98f181d2e5369ab273a5a3488ab7685d23c31ad5"}, + {file = "rapidfuzz-3.14.6.tar.gz", hash = "sha256:e13a8160d017b499ec7a2fa9d0ce1ae2e7377080815785819f966fb235d4eb60"}, +] + +[package.extras] +all = ["numpy"] + [[package]] name = "rdflib" version = "7.5.0" @@ -2333,18 +2466,18 @@ tests = ["pandas", "polars", "pyarrow", "pytest", "pytest-cov", "pytest-xdist", [[package]] name = "somef" -version = "0.11.2" +version = "0.11.5" description = "SOftware Metadata Extraction Framework: A tool for automatically extracting relevant software metadata from a source code repository (README, package files, etc)." optional = false python-versions = "<3.13,>=3.11" groups = ["main"] files = [ - {file = "somef-0.11.2-py3-none-any.whl", hash = "sha256:4e4bf7db17dccbf26f79f1113408548f120e79c44b20d6f7eb1f42761137bc43"}, - {file = "somef-0.11.2.tar.gz", hash = "sha256:5b781773e7d0dcfccfef8b7e805475066aa4fc570988368da67cd70caf688d28"}, + {file = "somef-0.11.5-py3-none-any.whl", hash = "sha256:2b00e96769ccce201ea7cf04f4662a7a56f9f29a8bbf50eb8fe8ca1d0fdeeedb"}, + {file = "somef-0.11.5.tar.gz", hash = "sha256:11fac2187cf5aaf8b8e82cc8d73564f5d85f5872f0859f4ed7705091dd15b9c1"}, ] [package.dependencies] -bibtexparser = ">=1.4.1,<2.0.0" +bibtexparser = ">=2.1.0,<3.0.0" bs4 = ">=0.0.1,<0.0.3" chardet = ">=5.2,<8.0" click = ">=8.1.7,<9.0.0" @@ -2363,12 +2496,13 @@ numpy = ">=1.26.3,<3.0.0" pandas = ">=2.1.4,<3.0.0" pytest = ">=8,<10" pyyaml = ">=6.0.2,<7.0.0" +rapidfuzz = ">=3.14.6,<4.0.0" rdflib = ">7.0.0" requests = ">=2.34.2,<3.0.0" scikit-learn = "1.5.0" scipy = ">=1.11.4,<2.0.0" textblob = ">=0.17.1,<0.21.0" -validators = ">=0.22.0,<0.23.0" +validators = ">=0.35,<0.37" xgboost = ">=2.0.3,<4.0.0" [[package]] @@ -2523,26 +2657,18 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "validators" -version = "0.22.0" +version = "0.36.0" description = "Python Data Validation for Humans™" optional = false -python-versions = ">=3.8" +python-versions = ">=3.11" groups = ["main"] files = [ - {file = "validators-0.22.0-py3-none-any.whl", hash = "sha256:61cf7d4a62bbae559f2e54aed3b000cea9ff3e2fdbe463f51179b92c58c9585a"}, - {file = "validators-0.22.0.tar.gz", hash = "sha256:77b2689b172eeeb600d9605ab86194641670cdb73b60afd577142a9397873370"}, + {file = "validators-0.36.0-py3-none-any.whl", hash = "sha256:c52ad00435e385b95fa4d6dd3081cb20c0660e7fd07b214523c8584dd2edba34"}, + {file = "validators-0.36.0.tar.gz", hash = "sha256:92ad9ed00bbed320b784cec0ea90b213cf08904cf605b3d0852cfa4108e32a2c"}, ] [package.extras] -docs-offline = ["myst-parser (>=2.0.0)", "pypandoc-binary (>=1.11)", "sphinx (>=7.1.1)"] -docs-online = ["mkdocs (>=1.5.2)", "mkdocs-git-revision-date-localized-plugin (>=1.2.0)", "mkdocs-material (>=9.2.6)", "mkdocstrings[python] (>=0.22.0)", "pyaml (>=23.7.0)"] -hooks = ["pre-commit (>=3.3.3)"] -package = ["build (>=1.0.0)", "twine (>=4.0.2)"] -runner = ["tox (>=4.11.1)"] -sast = ["bandit[toml] (>=1.7.5)"] -testing = ["pytest (>=7.4.0)"] -tooling = ["black (>=23.7.0)", "pyright (>=1.1.325)", "ruff (>=0.0.287)"] -tooling-extras = ["pyaml (>=23.7.0)", "pypandoc-binary (>=1.11)", "pytest (>=7.4.0)"] +crypto-eth-addresses = ["eth-hash[pycryptodome] (>=0.7.0)"] [[package]] name = "xgboost" @@ -2578,4 +2704,4 @@ scikit-learn = ["scikit-learn"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "3a9a8e808b2ff30907dd01c7ddd30311b0ff27dba54511ffe16ad111a46d116b" +content-hash = "b94d27a25de8741c03a4f15f69c807af3f1733ae4f930d489ba6b7d86473b419" diff --git a/pyproject.toml b/pyproject.toml index 171322f..e744ad6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rsmetacheck" -version = "0.3.6" +version = "0.3.7" description = "Detect metadata pitfalls in software repositories" authors = ["Anas El Hounsri"] readme = "README.md" @@ -9,7 +9,7 @@ packages = [{include = "rsmetacheck", from = "src"}] [tool.poetry.dependencies] python = ">=3.11,<3.13" requests = "*" - somef = "0.11.2" + somef = "0.11.5" [tool.poetry.scripts] rsmetacheck = "rsmetacheck.cli:cli" diff --git a/src/rsmetacheck/__init__.py b/src/rsmetacheck/__init__.py index aa15ff9..19f51e8 100644 --- a/src/rsmetacheck/__init__.py +++ b/src/rsmetacheck/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.6" +__version__ = "0.3.7" from .detect_pitfalls_main import main from .cli import cli diff --git a/src/rsmetacheck/scripts/pitfalls/p002.py b/src/rsmetacheck/scripts/pitfalls/p002.py index 4c3f934..dea7d78 100644 --- a/src/rsmetacheck/scripts/pitfalls/p002.py +++ b/src/rsmetacheck/scripts/pitfalls/p002.py @@ -1,88 +1,271 @@ -import re -from typing import Dict, Optional - - -def extract_license_from_file(somef_data: Dict) -> Optional[Dict[str, str]]: - """ - Extract license content from LICENSE file in SoMEF output. - Returns a dict with source and content, or None if not found. - """ - if "license" not in somef_data: - return None - - license_entries = somef_data["license"] - if not isinstance(license_entries, list): - return None - - for entry in license_entries: - if "source" in entry: - source = entry["source"] - if "license" in source.lower(): - if "result" in entry and "value" in entry["result"]: - return { - "source": source, - "content": entry["result"]["value"] - } - - return None - - -def check_license_template_placeholders(license_content: str) -> bool: - """ - Check if license content contains template placeholders like , , . - """ - if not license_content: - return False - - placeholder_patterns = [ - r'', - r'', - r'', - r'', - r'', - r'', - r'', - r'\[year\]', - r'\[fullname\]', - r'\[name\]', - r'\[copyright holder\]', - r'', - r'', - r'\[yyyy\]', - r'\[name of copyright owner\]', - ] - - content_lower = license_content.lower() - - for pattern in placeholder_patterns: - if re.search(pattern, content_lower): - return True - - return False - - -def detect_license_template_placeholders(somef_data: Dict, file_name: str) -> Dict: - """ - Detect license template placeholder pitfall for a single repository. - Returns detection result with pitfall info or None if no pitfall found. - """ - result = { - "has_pitfall": False, - "file_name": file_name, - "license_source": None, - "placeholders_found": False - } - - license_info = extract_license_from_file(somef_data) - - if license_info: - license_content = license_info["content"] - result["license_source"] = license_info["source"] - - has_template_placeholders = check_license_template_placeholders(license_content) - - if has_template_placeholders: - result["has_pitfall"] = True - result["placeholders_found"] = True - - return result \ No newline at end of file +import re +from typing import Dict, Optional + +from rsmetacheck.utils.fetch_utils import fetch_text, fetch_license_file + +# Licences whose text ships an appendix of *instructions to the licensor* +# ("How to Apply These Terms…", "APPENDIX: How to apply…"). The placeholder +# tokens there (, , [yyyy], [name of copyright owner], +# ) are part of the licence itself, not an unfilled field, so these +# families are never reported as P002. +EXEMPT_FAMILIES = {"gpl", "lgpl", "agpl", "apache", "mpl", "epl", "cddl"} + +# Short permissive licences where the copyright line is genuinely the author's +# to fill in. Only the operating header is inspected for these. +CHECK_FAMILIES = {"mit", "mit-0", "bsd", "0bsd", "isc", "x11", "zlib", "unlicense"} + +# Order matters: copyleft families first, because their text often quotes other +# licence names (e.g. GPLv3 mentions the Apache License in the compatibility +# section). +FAMILY_SIGNATURES = [ + ("agpl", r"gnu\s+affero\s+general\s+public\s+license"), + ("lgpl", r"gnu\s+lesser\s+general\s+public\s+license"), + ("gpl", r"gnu\s+general\s+public\s+license"), + ("apache", r"apache\s+license"), + ("mpl", r"mozilla\s+public\s+license"), + ("epl", r"eclipse\s+public\s+license"), + ("cddl", r"common\s+development\s+and\s+distribution\s+license"), + ("x11", r"except\s+as\s+contained\s+in\s+this\s+notice"), + ("zlib", r"origin\s+of\s+this\s+software\s+must\s+not\s+be\s+misrepresented"), + ("unlicense", r"free\s+and\s+unencumbered\s+software\s+released\s+into\s+the\s+public\s+domain"), + ("isc", r"permission\s+to\s+use,\s*copy,\s*modify,\s*and/?or\s+distribute"), + ("bsd", r"redistribution\s+and\s+use\s+in\s+source\s+and\s+binary\s+forms"), + ("mit", r"permission\s+is\s+hereby\s+granted,\s*free\s+of\s+charge"), + ("mit", r"\bmit\s+license\b"), + ("mit-0", r"\bmit\s+no\s+attribution\b"), +] + +# Fixed tokens that only ever appear in an unfilled template. +PLACEHOLDER_PATTERNS = [ + r'', + r'', + r'', + r'', + r'', + r'', + r'', + r'', + r']*>', + r'', + r'', + r'\[year\]', + r'\[yyyy\]', + r'\[fullname\]', + r'\[name\]', + r'\[owner\]', + r'\[author\]', + r'\[copyright holder\]', + r'\[name of copyright owner\]', + r'\[insert[^\]]*\]', + r'\{year\}', + r'\{yyyy\}', + r'\{fullname\}', + r'\{name\}', + r'\{author\}', + r'\{email\}', + r'\byyyy\b', +] + +# "Copyright (c) YEAR" / "" / "20XX" in the copyright line itself. +COPYRIGHT_LINE_STUB = re.compile( + r"copyright[^\n]{0,40}?(?:" + r"|\[year\]|\{year\}||\[yyyy\]|\byyyy\b|" + r"\byear\b|\b4-digit year\b|\b20XX\b|\bname of (?:author|copyright owner)\b" + r")", + re.IGNORECASE, +) + +# Everything below an anchor is the licence body (grant, conditions, warranty), +# not the copyright header. +HEADER_ANCHORS = [ + r"permission\s+is\s+hereby\s+granted", + r"permission\s+to\s+use,\s*copy", + r"redistribution\s+and\s+use", + r"this\s+software\s+is\s+provided", + r"the\s+software\s+is\s+provided", + r"origin\s+of\s+this\s+software", + r"free\s+and\s+unencumbered\s+software", +] + +HEADER_MAX_LINES = 25 + + +def extract_license_from_file(somef_data: Dict) -> Optional[Dict[str, str]]: + """ + Extract the LICENSE file content. + + SoMEF keeps only the SPDX id in the license entry's ``value``, so the actual + file text is fetched from the entry's ``source`` URL when available. + """ + license_entries = somef_data.get("license") + if not isinstance(license_entries, list): + return None + + for entry in license_entries: + if "source" not in entry: + continue + source = entry["source"] + source_str = str(source) + if not re.search(r'licen[cs]e|copying|copyright|unlicense', source_str, re.IGNORECASE): + continue + + if source_str.startswith(("http://", "https://")): + content = fetch_text(source_str) + if content: + return {"source": source_str, "content": content} + + result = entry.get("result") or {} + value = result.get("value") + if isinstance(value, str) and value.strip(): + return {"source": source_str, "content": value} + + # Fallback: SoMEF may have merged the license entries and dropped the + # LICENSE/COPYING source. Probe the repository root for a license file. + url, content = fetch_license_file(somef_data) + if url and content: + return {"source": url, "content": content} + + return None + + +def detect_license_family(license_content: str) -> str: + """Classify the licence family from its text. Returns "" when unknown. + + The title area is inspected first: GPLv3's compatibility section mentions the + GNU Affero GPL, so a whole-text scan would mislabel GPLv3 as AGPL. + """ + if not license_content: + return "" + + head = "\n".join(license_content.splitlines()[:8]).lower() + body = license_content.lower() + + for scope in (head, body): + for name, pattern in FAMILY_SIGNATURES: + if re.search(pattern, scope): + return _refine_family(name, body) + + return "" + + +def _refine_family(family: str, text: str) -> str: + """Disambiguate the BSD variants that share the ISC grant wording.""" + if family == "isc": + flat = re.sub(r"\s+", " ", text) + if re.search(r"zero[- ]clause|\b0bsd\b", flat): + return "0bsd" + # Without the retention clause the grant is 0BSD rather than ISC. + if "provided that the above copyright" not in flat: + return "0bsd" + return family + + +def operative_region(license_content: str, max_lines: int = HEADER_MAX_LINES) -> str: + """ + Return the copyright/attribution header of a licence: everything before the + body grant, capped at ``max_lines`` lines. + """ + lines = license_content.splitlines() + cut = len(lines) + for index, line in enumerate(lines): + if index >= max_lines: + break + lowered = line.lower() + if any(re.search(anchor, lowered) for anchor in HEADER_ANCHORS): + cut = index + break + return "\n".join(lines[:min(cut, max_lines)]) + + +def find_placeholders(region: str): + """Return the placeholder tokens found in ``region`` (deduplicated, ordered).""" + found = [] + lowered = region.lower() + for pattern in PLACEHOLDER_PATTERNS: + match = re.search(pattern, lowered) + if match: + token = match.group(0) + if token not in found: + found.append(token) + # A bare "YEAR"/"4-digit year" stub only matters when no explicit token was + # found; otherwise it just repeats the same evidence with prose attached. + if not found: + stub = COPYRIGHT_LINE_STUB.search(region) + if stub: + found.append(stub.group(0).strip()) + return found + + +def analyze_license_placeholders(license_content: str) -> Dict: + """ + Decide whether a licence file carries unfilled template placeholders. + + GPL/LGPL/AGPL/Apache/MPL/EPL/CDDL are exempt: their placeholder tokens belong + to the licence's own "how to apply" appendix. Permissive licences + (MIT/MIT-0/BSD/0BSD/ISC/X11/zlib/Unlicense) are checked in the copyright + header only, and an unrecognised licence is treated the same way so that + arbitrary prose cannot be mistaken for a template. + """ + analysis = { + "family": "", + "decision": "no_content", + "placeholders": [], + "has_pitfall": False, + "region": "", + } + if not license_content or not license_content.strip(): + return analysis + + family = detect_license_family(license_content) + analysis["family"] = family + + if family in EXEMPT_FAMILIES: + analysis["decision"] = "exempt_appendix_family" + return analysis + + region = operative_region(license_content) + analysis["region"] = region + placeholders = find_placeholders(region) + analysis["placeholders"] = placeholders + + if placeholders: + analysis["decision"] = "placeholder_in_copyright_header" + analysis["has_pitfall"] = True + else: + analysis["decision"] = "clean" + return analysis + + +def check_license_template_placeholders(license_content: str) -> bool: + """True when the licence's copyright header still carries template tokens.""" + return analyze_license_placeholders(license_content)["has_pitfall"] + + +def detect_license_template_placeholders(somef_data: Dict, file_name: str) -> Dict: + """ + Detect license template placeholder pitfall for a single repository. + """ + result = { + "has_pitfall": False, + "file_name": file_name, + "license_source": None, + "placeholders_found": False, + "license_family": None, + "placeholders": [], + "decision": None, + } + + license_info = extract_license_from_file(somef_data) + + if license_info: + analysis = analyze_license_placeholders(license_info["content"]) + result["license_source"] = license_info["source"] + result["license_family"] = analysis["family"] or None + result["placeholders"] = analysis["placeholders"] + result["decision"] = analysis["decision"] + + if analysis["has_pitfall"]: + result["has_pitfall"] = True + result["placeholders_found"] = True + + return result diff --git a/src/rsmetacheck/scripts/pitfalls/p005.py b/src/rsmetacheck/scripts/pitfalls/p005.py index a820a15..f83a316 100644 --- a/src/rsmetacheck/scripts/pitfalls/p005.py +++ b/src/rsmetacheck/scripts/pitfalls/p005.py @@ -1,66 +1,74 @@ -from typing import Dict -import re - -def is_software_archive_url(url: str) -> bool: - """ - Check if URL points to a software archive instead of a research paper. - """ - if not url or not isinstance(url, str): - return False - - url = url.lower().strip() - - # Patterns that indicate software archives - software_archive_patterns = [ - r'zenodo\.org', # Zenodo software archives - r'figshare\.com', # Figshare software archives - r'github\.com/.*/releases', # GitHub releases - r'sourceforge\.net', # SourceForge - r'archive\.org', # Internet Archive - r'codeocean\.com', # CodeOcean - r'osf\.io', # Open Science Framework (when used for software) - r'doi\.org/10\.5281', # Zenodo DOIs - ] - - for pattern in software_archive_patterns: - if re.search(pattern, url): - return True - - return False - - -def detect_reference_publication_archive_pitfall(somef_data: Dict, file_name: str) -> Dict: - """ - Detect when codemeta.json referencePublication refers to software archive instead of paper. - """ - result = { - "has_pitfall": False, - "file_name": file_name, - "reference_url": None, - "source": None, - "is_software_archive": False - } - - if "reference_publication" not in somef_data: - return result - - ref_pub_entries = somef_data["reference_publication"] - if not isinstance(ref_pub_entries, list): - return result - - for entry in ref_pub_entries: - source = entry.get("source", "") - technique = entry.get("technique", "") - - if "codemeta.json" in source or (technique == "code_parser" and "codemeta" in source.lower()): - if "result" in entry and "value" in entry["result"]: - ref_url = entry["result"]["value"] - - if is_software_archive_url(ref_url): - result["has_pitfall"] = True - result["reference_url"] = ref_url - result["source"] = source - result["is_software_archive"] = True - break - +from typing import Dict +import re + +def is_software_archive_url(url: str) -> bool: + """ + Check if URL points to a software archive instead of a research paper. + """ + if not url or not isinstance(url, str): + return False + + url = url.lower().strip() + + # Patterns that indicate software archives + software_archive_patterns = [ + r'zenodo\.org', # Zenodo software archives + r'figshare\.com', # Figshare software archives + r'github\.com/.*/releases', # GitHub releases + r'sourceforge\.net', # SourceForge + r'archive\.org', # Internet Archive + r'codeocean\.com', # CodeOcean + r'osf\.io', # Open Science Framework (when used for software) + r'doi\.org/10\.5281', # Zenodo DOIs + r'10\.5281/zenodo\.', # bare Zenodo DOI (10.5281/zenodo.xxx) + ] + + for pattern in software_archive_patterns: + if re.search(pattern, url): + return True + + return False + + +def detect_reference_publication_archive_pitfall(somef_data: Dict, file_name: str) -> Dict: + """ + Detect when codemeta.json referencePublication refers to software archive instead of paper. + + SoMEF 0.11.2 stores the codemeta.json ``referencePublication`` value under the + ``citation`` category (there is no ``reference_publication`` key in the output), + so this detector reads ``citation`` entries sourced from codemeta.json. + """ + result = { + "has_pitfall": False, + "file_name": file_name, + "reference_url": None, + "source": None, + "is_software_archive": False + } + + citation_entries = somef_data.get("citation") + if not isinstance(citation_entries, list): + citation_entries = [] + + legacy_entries = somef_data.get("reference_publication") + if isinstance(legacy_entries, list): + citation_entries = citation_entries + legacy_entries + + for entry in citation_entries: + source = entry.get("source", "") + technique = entry.get("technique", "") + source_str = str(source) + + if "codemeta.json" in source_str.lower() or ( + technique == "code_parser" and "codemeta" in source_str.lower()): + result_data = entry.get("result") or {} + ref_url = result_data.get("value") or result_data.get("url") or result_data.get("doi") + + if isinstance(ref_url, str) and is_software_archive_url(ref_url): + result["has_pitfall"] = True + result["reference_url"] = ref_url + result["source"] = source + result["is_software_archive"] = True + break + return result \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p006.py b/src/rsmetacheck/scripts/pitfalls/p006.py index 1c401af..e65d192 100644 --- a/src/rsmetacheck/scripts/pitfalls/p006.py +++ b/src/rsmetacheck/scripts/pitfalls/p006.py @@ -1,5 +1,9 @@ from typing import Dict +import re + from rsmetacheck.utils.pitfall_utils import extract_metadata_source_filename +from rsmetacheck.utils.fetch_utils import find_source_url, fetch_text + def is_local_file_license(license_value: str) -> bool: """ @@ -8,10 +12,15 @@ def is_local_file_license(license_value: str) -> bool: if not license_value or not isinstance(license_value, str): return False + license_value = re.sub(r'^\s*licen[cs]e\s+file\s*:?\s*', '', license_value, flags=re.IGNORECASE) + license_lower = license_value.lower().strip() if license_lower.startswith('http://') or license_lower.startswith('https://'): return False + if 'http' in license_lower: + # e.g. "GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt" names the license + return False if license_value.startswith('./') or license_value.startswith('../'): return True @@ -65,7 +74,13 @@ def detect_local_file_license_pitfall(somef_data: Dict, file_name: str) -> Dict: technique = entry.get("technique", "") source = entry.get("source", "") - if technique == "code_parser" or any(src in source.lower() for src in metadata_sources): + # Require the licence value to come from a metadata manifest. The bare + # `technique == "code_parser"` branch also matched non-manifest parsers + # (e.g. SoMEF >= 0.11.5 parses .zenodo.json), which reports licence + # identifiers rather than a path/name that this check is about. Entries + # that carry no provenance at all are still accepted. + has_known_source = any(src in source.lower() for src in metadata_sources) + if has_known_source or (technique == "code_parser" and not source.strip()): if "result" in entry and "value" in entry["result"]: license_value = entry["result"]["value"] @@ -79,4 +94,26 @@ def detect_local_file_license_pitfall(somef_data: Dict, file_name: str) -> Dict: result["has_pitfall"] = True result["is_local_file"] = True + if not result["has_pitfall"]: + pyproject_url = find_source_url(somef_data, "pyproject.toml") + if pyproject_url: + text = fetch_text(pyproject_url) + if text: + license_match = re.search(r'^\s*license\s*=\s*(.*)$', text, re.MULTILINE) + if license_match: + raw = license_match.group(1).strip() + file_match = re.search(r'file\s*=\s*["\']([^"\']+)["\']', raw) + if file_match: + value = file_match.group(1) + else: + simple = re.search(r'["\']([^"\']+)["\']', raw) + value = simple.group(1) if simple else raw + if value and is_local_file_license(value): + result["has_pitfall"] = True + result["is_local_file"] = True + result["license_value"] = value + result["source"] = pyproject_url + result["metadata_source_file"] = "pyproject.toml" + result["metadata_source_files"].append("pyproject.toml") + return result \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p008.py b/src/rsmetacheck/scripts/pitfalls/p008.py index a675d4c..2bb9df5 100644 --- a/src/rsmetacheck/scripts/pitfalls/p008.py +++ b/src/rsmetacheck/scripts/pitfalls/p008.py @@ -1,8 +1,10 @@ from typing import Dict import requests import re +import time from urllib.parse import urlparse from rsmetacheck.utils.pitfall_utils import extract_metadata_source_filename +from rsmetacheck.utils.fetch_utils import classify_status, HARD_FAILURE def is_valid_url_format(url: str) -> bool: @@ -27,40 +29,78 @@ def is_valid_url_format(url: str) -> bool: def check_url_status(url: str, timeout: int = 10) -> Dict: """ - Check if URL returns a valid response (not 404 or other error). + Check if URL returns a valid response. + + Only definitive failures (404/410 and DNS/connection errors) are treated as + invalid; transient failures (rate limiting, timeouts, server errors) are + marked inconclusive so the detector does not produce false positives. """ result = { "is_accessible": False, "status_code": None, - "error": None + "error": None, + "verdict": "inconclusive", } if not is_valid_url_format(url): result["error"] = "Invalid URL format" + result["verdict"] = HARD_FAILURE return result if url.startswith(('git+', 'git://', 'svn+', 'hg+', 'bzr+')): result["is_accessible"] = True result["status_code"] = 200 + result["verdict"] = "accessible" return result - try: - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - } + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + } + + status_code = None + error = None + for attempt in range(3): + try: + response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) + status_code = response.status_code + error = None + if status_code == 429 and attempt < 2: + time.sleep(1 + attempt) + continue + break + except requests.exceptions.RequestException as e: + status_code = None + error = str(e) + if attempt < 2: + time.sleep(0.5) + continue + break + except Exception as e: + status_code = None + error = f"Unexpected error: {str(e)}" + break + + result["status_code"] = status_code + result["error"] = error + result["verdict"] = classify_status(status_code, error) + result["is_accessible"] = result["verdict"] == "accessible" - response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) - result["status_code"] = response.status_code + return result - if (200 <= response.status_code < 300) or response.status_code == 301: - result["is_accessible"] = True - except requests.exceptions.RequestException as e: - result["error"] = str(e) - except Exception as e: - result["error"] = f"Unexpected error: {str(e)}" +def _is_invalid_url(url_status: Dict) -> bool: + """True if a URL is definitively invalid (hard failure). - return result + Accepts both the new status dict (with a ``verdict``) and the legacy shape + (only ``status_code``/``error``) produced by older test doubles. + """ + verdict = url_status.get("verdict") + if verdict: + return verdict == HARD_FAILURE + if url_status.get("status_code") in (404, 410): + return True + error = url_status.get("error") + return isinstance(error, str) and "Invalid URL format" in error def extract_urls_from_requirements(requirement_text: str) -> list: @@ -130,7 +170,7 @@ def detect_invalid_software_requirement_pitfall(somef_data: Dict, file_name: str if isinstance(req_value, str) and is_valid_url_format(req_value): url_status = check_url_status(req_value) - if not url_status["is_accessible"]: + if _is_invalid_url(url_status): source_filename = extract_metadata_source_filename(source) if result["source"] is None: result["source"] = source @@ -163,7 +203,7 @@ def detect_invalid_software_requirement_pitfall(somef_data: Dict, file_name: str for url in urls: url_status = check_url_status(url) - if not url_status["is_accessible"]: + if _is_invalid_url(url_status): invalid_urls.append({ "url": url, "status_code": url_status["status_code"], diff --git a/src/rsmetacheck/scripts/pitfalls/p010.py b/src/rsmetacheck/scripts/pitfalls/p010.py index 0ae47b5..576bbf3 100644 --- a/src/rsmetacheck/scripts/pitfalls/p010.py +++ b/src/rsmetacheck/scripts/pitfalls/p010.py @@ -1,129 +1,160 @@ -import re -from typing import Dict, Optional - - -def extract_license_from_file(somef_data: Dict) -> Optional[Dict[str, str]]: - """ - Extract license content from LICENSE file in SoMEF output. - Returns a dict with source and content, or None if not found. - """ - if "license" not in somef_data: - return None - - license_entries = somef_data["license"] - if not isinstance(license_entries, list): - return None - - for entry in license_entries: - if "source" in entry: - source = entry["source"] - if "LICENSE" in source.upper() and "result" in entry and "value" in entry["result"]: - return { - "source": source, - "content": entry["result"]["value"] - } - - return None - - -def check_copyright_only_license(license_content: str) -> bool: - """ - Check if license file only contains copyright information without actual license terms. - Example: - YEAR: 2017 - COPYRIGHT HOLDER: Adam H. Sparks - """ - if not license_content: - return False - - content_lower = license_content.lower().strip() - content_lines = [line.strip() for line in license_content.strip().split('\n') if line.strip()] - - copyright_only_patterns = [ - r'year\s*:\s*\d{4}', # YEAR: 2017 - r'copyright\s+holder\s*:\s*[a-zA-Z]', # COPYRIGHT HOLDER: Someone - r'author\s*:\s*[a-zA-Z]', # AUTHOR: Someone - r'copyright\s*©?\s*\d{4}', # Copyright 2017 or Copyright © 2017 - r'\(c\)\s*\d{4}', # (C) 2017 - ] - - license_term_patterns = [ - r'permission\s+is\s+hereby\s+granted', - r'subject\s+to\s+the\s+following\s+conditions', - r'redistribution\s+and\s+use', - r'without\s+restriction', - r'without\s+warranty', - r'liability', - r'terms\s+and\s+conditions', - r'licensed\s+under', - r'mit\s+license', - r'apache\s+license', - r'gnu\s+general\s+public\s+license', - r'bsd\s+license', - r'creative\s+commons', - ] - - has_copyright_info = any(re.search(pattern, content_lower) for pattern in copyright_only_patterns) - has_license_terms = any(re.search(pattern, content_lower) for pattern in license_term_patterns) - - if has_license_terms: - return False - - # This will check if it has copyright info but no license terms and is short, it's likely copyright-only - if has_copyright_info and not has_license_terms and len(content_lines) <= 10: - return True - - # Check for the exact format "YEAR: xxxx" and "COPYRIGHT HOLDER: xxxx" - year_pattern_found = bool(re.search(r'year\s*:\s*\d{4}', content_lower)) - copyright_holder_pattern_found = bool(re.search(r'copyright\s+holder\s*:', content_lower)) - - if year_pattern_found and copyright_holder_pattern_found: - if has_license_terms: - return False - return True - - if len(content_lines) <= 5: - meaningful_lines = [] - - for line in content_lines: - line_lower = line.lower() - - if not any(re.search(pattern, line_lower) for pattern in copyright_only_patterns): - - if (len(line.strip()) > 0 and - not line.strip().startswith('#') and - not line.strip().startswith('//') and - line.strip() not in ['', '-', '=', '*']): - meaningful_lines.append(line) - - if len(meaningful_lines) <= 1 and has_copyright_info: - return True - - return False - - -def detect_copyright_only_license(somef_data: Dict, file_name: str) -> Dict: - """ - Detect copyright-only license pitfall for a single repository. - Returns detection result with pitfall info or None if no pitfall found. - """ - result = { - "has_pitfall": False, - "file_name": file_name, - "license_source": None, - "is_copyright_only": False - } - - license_info = extract_license_from_file(somef_data) - - if license_info: - license_content = license_info["content"] - result["license_source"] = license_info["source"] - - is_copyright_only = check_copyright_only_license(license_content) - - if is_copyright_only: - result["has_pitfall"] = True - result["is_copyright_only"] = True - - return result \ No newline at end of file +import re +from typing import Dict, Optional + +from rsmetacheck.utils.fetch_utils import fetch_text, fetch_license_file + + +def extract_license_from_file(somef_data: Dict) -> Optional[Dict[str, str]]: + """ + Extract the first license file entry's content from SoMEF output. + + Kept for backward compatibility (the detector itself scans *all* license + entries via ``detect_copyright_only_license``). + """ + license_entries = somef_data.get("license") + if not isinstance(license_entries, list): + return None + + for entry in license_entries: + source = entry.get("source") + source_str = str(source) if source is not None else "" + if "LICENSE" not in source_str.upper(): + continue + result = entry.get("result") or {} + value = result.get("value") + if not value: + return None + return {"source": source_str, "content": value} + + return None + + +def detect_copyright_only_license(somef_data: Dict, file_name: str) -> Dict: + """ + Detect copyright-only license pitfall for a single repository. + Returns detection result with pitfall info or None if no pitfall found. + + Every license file entry is evaluated (a repository may ship both a + copyright-only ``LICENSE`` and a full ``LICENSE.md``), and the file text is + fetched from the source URL when SoMEF only kept the SPDX id. + """ + result = { + "has_pitfall": False, + "file_name": file_name, + "license_source": None, + "is_copyright_only": False + } + + license_entries = somef_data.get("license") + if not isinstance(license_entries, list): + return result + + for entry in license_entries: + source = entry.get("source") + source_str = str(source) if source else "" + if not re.search(r'licen[cs]e|copying|copyright|unlicense', source_str, re.IGNORECASE): + continue + + if result["license_source"] is None: + result["license_source"] = source_str + + content = None + if source_str.startswith(("http://", "https://")): + content = fetch_text(source_str) + if content is None: + value = (entry.get("result") or {}).get("value") + if isinstance(value, str): + content = value + + if content and check_copyright_only_license(content): + result["has_pitfall"] = True + result["is_copyright_only"] = True + result["license_source"] = source_str or content[:120] + break + + # Fallback: no license-file entry was available in the SoMEF output. + if result["license_source"] is None: + url, content = fetch_license_file(somef_data) + if url and content: + result["license_source"] = url + if check_copyright_only_license(content): + result["has_pitfall"] = True + result["is_copyright_only"] = True + + return result + + +def check_copyright_only_license(license_content: str) -> bool: + """ + Check if license file only contains copyright information without actual license terms. + Example: + YEAR: 2017 + COPYRIGHT HOLDER: Adam H. Sparks + """ + if not license_content: + return False + + content_lower = license_content.lower().strip() + content_lines = [line.strip() for line in license_content.strip().split('\n') if line.strip()] + + copyright_only_patterns = [ + r'year\s*:\s*\d{4}', # YEAR: 2017 + r'copyright\s+holder\s*:\s*[a-zA-Z]', # COPYRIGHT HOLDER: Someone + r'author\s*:\s*[a-zA-Z]', # AUTHOR: Someone + r'copyright\s*©?\s*\d{4}', # Copyright 2017 or Copyright © 2017 + r'\(c\)\s*\d{4}', # (C) 2017 + ] + + license_term_patterns = [ + r'permission\s+is\s+hereby\s+granted', + r'subject\s+to\s+the\s+following\s+conditions', + r'redistribution\s+and\s+use', + r'without\s+restriction', + r'without\s+warranty', + r'liability', + r'terms\s+and\s+conditions', + r'licensed\s+under', + r'mit\s+license', + r'apache\s+license', + r'gnu\s+general\s+public\s+license', + r'bsd\s+license', + r'creative\s+commons', + ] + + has_copyright_info = any(re.search(pattern, content_lower) for pattern in copyright_only_patterns) + has_license_terms = any(re.search(pattern, content_lower) for pattern in license_term_patterns) + + if has_license_terms: + return False + + # This will check if it has copyright info but no license terms and is short, it's likely copyright-only + if has_copyright_info and not has_license_terms and len(content_lines) <= 10: + return True + + # Check for the exact format "YEAR: xxxx" and "COPYRIGHT HOLDER: xxxx" + year_pattern_found = bool(re.search(r'year\s*:\s*\d{4}', content_lower)) + copyright_holder_pattern_found = bool(re.search(r'copyright\s+holder\s*:', content_lower)) + + if year_pattern_found and copyright_holder_pattern_found: + if has_license_terms: + return False + return True + + if len(content_lines) <= 5: + meaningful_lines = [] + + for line in content_lines: + line_lower = line.lower() + + if not any(re.search(pattern, line_lower) for pattern in copyright_only_patterns): + + if (len(line.strip()) > 0 and + not line.strip().startswith('#') and + not line.strip().startswith('//') and + line.strip() not in ['', '-', '=', '*']): + meaningful_lines.append(line) + + if len(meaningful_lines) <= 1 and has_copyright_info: + return True + + return False \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p011.py b/src/rsmetacheck/scripts/pitfalls/p011.py index e3aeb3a..245c68d 100644 --- a/src/rsmetacheck/scripts/pitfalls/p011.py +++ b/src/rsmetacheck/scripts/pitfalls/p011.py @@ -1,6 +1,11 @@ from typing import Dict +import time from urllib.parse import urlparse +import requests + +from rsmetacheck.utils.fetch_utils import classify_status, HARD_FAILURE + def is_valid_issue_tracker_format(url: str) -> bool: """ @@ -35,17 +40,53 @@ def is_valid_issue_tracker_format(url: str) -> bool: return False +def _is_unreachable(url: str, timeout: int = 10) -> Dict: + """Return status/error for an issue tracker URL (only hard failures matter).""" + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + } + status_code = None + error = None + for attempt in range(3): + try: + response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) + status_code = response.status_code + error = None + if status_code == 429 and attempt < 2: + time.sleep(1 + attempt) + continue + break + except requests.exceptions.RequestException as e: + status_code = None + error = str(e) + if attempt < 2: + time.sleep(0.5) + continue + break + except Exception as e: + status_code = None + error = f"Unexpected error: {str(e)}" + break + return { + "status_code": status_code, + "error": error, + "verdict": classify_status(status_code, error), + } + + def detect_issue_tracker_format_pitfall(somef_data: Dict, file_name: str) -> Dict: """ - Detect when codemeta.json IssueTracker URL does not follow - a recognized issue tracker format. + Detect when codemeta.json IssueTracker URL does not follow a recognized + format, or is a dead link (404/410/unresolvable host). """ result = { "has_pitfall": False, "file_name": file_name, "issue_url": None, "source": None, - "format_violation": None + "format_violation": None, + "status_code": None, + "error": None, } if "issue_tracker" not in somef_data: @@ -70,4 +111,14 @@ def detect_issue_tracker_format_pitfall(somef_data: Dict, file_name: str) -> Dic result["format_violation"] = "URL does not match recognized issue tracker format" break + url_status = _is_unreachable(issue_url) + if url_status.get("verdict") == HARD_FAILURE: + result["has_pitfall"] = True + result["issue_url"] = issue_url + result["source"] = source + result["status_code"] = url_status["status_code"] + result["error"] = url_status["error"] + result["format_violation"] = "Issue tracker URL is not reachable" + break + return result \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p012.py b/src/rsmetacheck/scripts/pitfalls/p012.py index d78c995..6de2c2b 100644 --- a/src/rsmetacheck/scripts/pitfalls/p012.py +++ b/src/rsmetacheck/scripts/pitfalls/p012.py @@ -14,6 +14,8 @@ def extract_version_from_download_url(url: str) -> str: r'/archive/(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)\.', # /archive/3.8.0. or /archive/v1.2.3. r'/archive/(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)$', # /archive/3.8.0 or /archive/v1.2.3 (end of string) + r'/refs/tags/(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)', # /archive/refs/tags/v1.2.3.zip + r'/releases/tag/(?:[^/]*?[-_])?(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)$', # /releases/tag/name-1.2.3 r'[-_](?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)\.', # -3.8.0.tar.gz or _v1.2.3.zip r'/(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)/[^/]*$', # /3.8.0/something ] @@ -67,7 +69,9 @@ def get_latest_release_version(somef_data: Dict) -> str: if "tag" in result and result["tag"]: tag = result["tag"].strip() if tag: - return normalize_version(tag) + version_match = re.search(r'(?:v)?(\d+\.\d+(?:\.\d+)?(?:[a-zA-Z0-9\-\.]*)?)', tag) + if version_match: + return normalize_version(version_match.group(1)) if "name" in result and result["name"]: name = result["name"] @@ -119,14 +123,16 @@ def detect_outdated_download_url_pitfall(somef_data: Dict, file_name: str) -> Di if not download_version: return result + normalized_download_version = normalize_version(download_version) + if not normalized_download_version: + return result + latest_version = get_latest_release_version(somef_data) if not latest_version: return result - normalized_download_version = normalize_version(download_version) normalized_latest_version = normalize_version(latest_version) - - if not normalized_download_version or not normalized_latest_version: + if not normalized_latest_version: return result if normalized_download_version != normalized_latest_version: diff --git a/src/rsmetacheck/scripts/pitfalls/p013.py b/src/rsmetacheck/scripts/pitfalls/p013.py index 1100a51..828be57 100644 --- a/src/rsmetacheck/scripts/pitfalls/p013.py +++ b/src/rsmetacheck/scripts/pitfalls/p013.py @@ -27,12 +27,15 @@ def detect_license_no_version_pitfall(somef_data: Dict, file_name: str) -> Dict: metadata_sources = ["codemeta.json", "DESCRIPTION", "composer.json", "package.json", "pom.xml", "pyproject.toml", "requirements.txt", "setup.py"] + # Optional "v"/"version" token so that "GPL v3.0", "LGPL v3.0" etc. are + # correctly recognized as versioned (not flagged as missing a version). + _V = r'(?:\s*v(?:ersion)?\.?\s*)?' versioned_patterns = { - "GPL": r"\bGPL[-\s]?\(?\s*(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", - "LGPL": r"\bLGPL[-\s]?\(?\s*(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", - "AGPL": r"\bAGPL[-\s]?\(?\s*(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", - "Apache": r"\bApache[-\s]?\(?\s*(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", - "CC": r"\bCC[- ]BY[-\s]?\(?\s*(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", + "GPL": rf"\bGPL[-\s]?\(?{_V}(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", + "LGPL": rf"\bLGPL[-\s]?\(?{_V}(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", + "AGPL": rf"\bAGPL[-\s]?\(?{_V}(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", + "Apache": rf"\bApache[-\s]?\(?{_V}(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", + "CC": rf"\bCC[- ]BY[-\s]?\(?{_V}(?:>=?|<=?|>|<|=)?\s*\d+(\.\d+)?\)?", "BSD": r"\bBSD[-\s]\d+[-\s]Clause" } diff --git a/src/rsmetacheck/scripts/pitfalls/p014.py b/src/rsmetacheck/scripts/pitfalls/p014.py index ff11f8a..a8fb9bf 100644 --- a/src/rsmetacheck/scripts/pitfalls/p014.py +++ b/src/rsmetacheck/scripts/pitfalls/p014.py @@ -55,11 +55,18 @@ def detect_bare_doi_pitfall(somef_data: Dict, file_name: str) -> Dict: if "result" in entry and "value" in entry["result"]: identifier_value = entry["result"]["value"] - if is_bare_doi(identifier_value): - result["has_pitfall"] = True - result["identifier_value"] = identifier_value - result["source"] = source - result["is_bare_doi"] = True + # SoMEF < 0.11.5 kept an identifier array as a single list value; + # >= 0.11.5 emits one entry per element. Accept both shapes. + candidates = identifier_value if isinstance(identifier_value, list) else [identifier_value] + + for candidate in candidates: + if is_bare_doi(candidate): + result["has_pitfall"] = True + result["identifier_value"] = candidate + result["source"] = source + result["is_bare_doi"] = True + break + if result["has_pitfall"]: break return result \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p015.py b/src/rsmetacheck/scripts/pitfalls/p015.py index 03d8fce..443cd3d 100644 --- a/src/rsmetacheck/scripts/pitfalls/p015.py +++ b/src/rsmetacheck/scripts/pitfalls/p015.py @@ -1,89 +1,400 @@ -from typing import Dict -import requests -from urllib.parse import urlparse - - -def is_valid_url_format(url: str) -> bool: - """ - Check if URL has a valid format. - """ - try: - result = urlparse(url) - return all([result.scheme, result.netloc]) - except: - return False - - -def check_ci_url_status(url: str, timeout: int = 10) -> Dict: - """ - Check if continuous integration URL returns a valid response. - """ - result = { - "is_accessible": False, - "status_code": None, - "error": None - } - - if not is_valid_url_format(url): - result["error"] = "Invalid URL format" - return result - - try: - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - } - - response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) - result["status_code"] = response.status_code - - # Consider 200-302 (except 300) as successful - if 200 <= response.status_code < 300 or 300 < response.status_code <303 : - result["is_accessible"] = True - - except requests.exceptions.RequestException as e: - result["error"] = str(e) - except Exception as e: - result["error"] = f"Unexpected error: {str(e)}" - - return result - - -def detect_ci_404_pitfall(somef_data: Dict, file_name: str) -> Dict: - """ - Detect when codemeta.json continuous integration link returns 404. - """ - result = { - "has_pitfall": False, - "file_name": file_name, - "ci_url": None, - "source": None, - "status_code": None, - "error": None - } - - if "continuous_integration" not in somef_data: - return result - - ci_entries = somef_data["continuous_integration"] - if not isinstance(ci_entries, list): - return result - - for entry in ci_entries: - source = entry.get("source", "") - technique = entry.get("technique", "") - - if "codemeta.json" in source or (technique == "code_parser" and "codemeta" in source.lower()): - if "result" in entry and "value" in entry["result"]: - ci_url = entry["result"]["value"] - - url_status = check_ci_url_status(ci_url) - - if not url_status["is_accessible"]: - result["has_pitfall"] = True - result["ci_url"] = ci_url - result["source"] = source - result["status_code"] = url_status["status_code"] - result["error"] = url_status["error"] - break - - return result \ No newline at end of file +from typing import Dict +import json +import os +import time +from urllib.parse import urlparse, quote + +import requests + +from rsmetacheck.utils.fetch_utils import classify_status, HARD_FAILURE, find_source_url, fetch_text + +GITHUB_HOSTS = {"github.com", "www.github.com"} +_TOKEN_ENV_VARS = ("GITHUB_API_TOKEN", "GITHUB_TOKEN", "GH_TOKEN") + + +def is_valid_url_format(url: str) -> bool: + """ + Check if URL has a valid format. + """ + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except: + return False + + +# ---------------------------------------------------------------- GitHub API path +# +# GitHub serves /actions/... as a client-side route: the HTML page answers 200 +# even when the workflow file does not exist, so an HTTP check cannot tell a +# live CI link from a dead one. GitHub-hosted Actions URLs are therefore +# resolved through the REST API, which answers 404 for a missing workflow. + +def parse_github_actions_url(url: str): + """ + Describe the GitHub Actions target behind a URL, or return None when the URL + is not a GitHub Actions link. + + Supported shapes: + /{owner}/{repo}/actions/workflows/{file} + /{owner}/{repo}/actions + /{owner}/{repo}/workflows/{file}/badge.svg + /{owner}/{repo}/blob/{ref}/.github/workflows/{file} + /{owner}/{repo}/tree/{ref}/.github/workflows + """ + if not url or not isinstance(url, str): + return None + try: + parsed = urlparse(url.strip()) + except Exception: + return None + if parsed.netloc.lower() not in GITHUB_HOSTS: + return None + + parts = [p for p in parsed.path.split("/") if p] + if len(parts) < 2: + return None + owner, repo, rest = parts[0], parts[1], parts[2:] + + if rest[:2] == ["actions", "workflows"]: + if len(rest) >= 3: + return {"kind": "workflow_file", "owner": owner, "repo": repo, "file": rest[2]} + return {"kind": "actions_list", "owner": owner, "repo": repo} + + if rest[:1] == ["actions"]: + return {"kind": "actions_list", "owner": owner, "repo": repo} + + if rest[:1] == ["workflows"] and len(rest) >= 3 and rest[2].startswith("badge"): + return {"kind": "workflow_file", "owner": owner, "repo": repo, + "file": rest[1], "badge": True} + + # /blob//.github/workflows[/] and /tree//.github/workflows + if rest[:1] in (["blob"], ["tree"]) and len(rest) >= 4 and rest[2:4] == [".github", "workflows"]: + kind = "workflow_blob" if rest[0] == "blob" else "tree_workflows" + target = {"kind": kind, "owner": owner, "repo": repo, "ref": rest[1]} + if len(rest) >= 5: + target["file"] = "/".join(rest[4:]) + return target + + return None + + +def _github_token(): + """Token for API calls, conventional env vars first (the bot sets GITHUB_API_TOKEN).""" + for var in _TOKEN_ENV_VARS: + value = os.environ.get(var) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _safe_json(response): + try: + data = response.json() + except Exception: + return None + return data if isinstance(data, dict) else None + + +def _github_api_get(path: str, timeout: int = 10) -> Dict: + """GET api.github.com{path}; never raises, returns status/payload/error.""" + url = "https://api.github.com" + path + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} + token = _github_token() + if token: + headers["Authorization"] = f"token {token}" + + status_code = None + error = None + for attempt in range(3): + try: + response = requests.get(url, headers=headers, timeout=timeout) + status_code = response.status_code + error = None + if status_code == 429 and attempt < 2: + time.sleep(1 + attempt) + continue + return {"status_code": status_code, "payload": _safe_json(response), "error": None} + except requests.exceptions.RequestException as exc: + status_code = None + error = str(exc) + if attempt < 2: + time.sleep(0.5) + continue + break + except Exception as exc: # noqa: BLE001 - defers to inconclusive + status_code = None + error = f"Unexpected error: {exc}" + break + return {"status_code": status_code, "payload": None, "error": error} + + +def check_github_ci_status(url: str, timeout: int = 10) -> Dict: + """ + Resolve a GitHub Actions CI URL through the REST API. + + hard_failure only when the API states the workflow does not exist while the + repository itself is readable; a private/missing repository or a rate limit + stays inconclusive so the check cannot blame CI for an access problem. + """ + result = { + "is_accessible": False, + "status_code": None, + "error": None, + "verdict": "inconclusive", + "check_method": "github_api", + "api_status": None, + "repo_status": None, + "workflow_state": None, + "target": None, + } + + target = parse_github_actions_url(url) + if target is None: + result["error"] = "Not a GitHub Actions URL" + return result + + result["target"] = target + owner, repo = target["owner"], target["repo"] + + def repo_readable() -> bool: + probe = _github_api_get(f"/repos/{owner}/{repo}", timeout=timeout) + result["repo_status"] = probe["status_code"] + return probe["status_code"] == 200 + + def hard_failure(message: str, status_code=None): + result["verdict"] = HARD_FAILURE + result["status_code"] = status_code + result["error"] = message + return result + + def accessible(state=None): + result["verdict"] = "accessible" + result["is_accessible"] = True + if state is not None: + result["workflow_state"] = state + return result + + kind = target["kind"] + + if kind == "workflow_file": + file_name = target["file"] + response = _github_api_get( + f"/repos/{owner}/{repo}/actions/workflows/{quote(file_name, safe='')}", timeout=timeout) + result["api_status"] = response["status_code"] + if response["status_code"] == 200: + return accessible((response["payload"] or {}).get("state")) + if response["status_code"] in (404, 410): + if repo_readable(): + return hard_failure( + f"workflow '{file_name}' does not exist in {owner}/{repo}", + response["status_code"]) + return result # repository missing or private: not CI's fault + if response["status_code"] in (401, 403, 429): + result["error"] = "GitHub API rate limited or unauthorised" + return result + result["error"] = response["error"] + return result + + if kind == "workflow_blob": + file_name = target.get("file") + path = f"/repos/{owner}/{repo}/contents/.github/workflows" + if file_name: + path += "/" + quote(file_name, safe="/") + query = f"?ref={quote(target['ref'], safe='')}" if target.get("ref") else "" + response = _github_api_get(path + query, timeout=timeout) + result["api_status"] = response["status_code"] + if response["status_code"] == 200: + return accessible() + if response["status_code"] in (404, 410): + if repo_readable(): + return hard_failure( + f"workflow path '{file_name or '.github/workflows'}' does not exist on ref " + f"'{target.get('ref')}'", response["status_code"]) + return result + result["error"] = response["error"] + return result + + if kind == "tree_workflows": + query = f"?ref={quote(target['ref'], safe='')}" if target.get("ref") else "" + response = _github_api_get( + f"/repos/{owner}/{repo}/contents/.github/workflows{query}", timeout=timeout) + result["api_status"] = response["status_code"] + if response["status_code"] == 200: + payload = response["payload"] + entries = payload if isinstance(payload, list) else None + if entries is not None and len(entries) == 0: + return hard_failure("no workflows present in .github/workflows", 200) + return accessible() + if response["status_code"] in (404, 410) and repo_readable(): + return hard_failure("'.github/workflows' does not exist in the repository", + response["status_code"]) + result["error"] = response["error"] + return result + + # actions_list: the repository Actions page itself + response = _github_api_get(f"/repos/{owner}/{repo}/actions/workflows", timeout=timeout) + result["api_status"] = response["status_code"] + if response["status_code"] == 200: + total = (response["payload"] or {}).get("total_count") + if isinstance(total, int) and total == 0: + return hard_failure("repository has no workflows configured", 200) + return accessible() + if response["status_code"] in (404, 410) and repo_readable(): + return hard_failure("repository has no Actions enabled", response["status_code"]) + result["error"] = response["error"] + return result + + +# ---------------------------------------------------------------- HTTP path + +def _check_ci_url_status_html(url: str, timeout: int = 10) -> Dict: + """ + Check a non-GitHub CI URL over HTTP. + + Only definitive failures (404/410/DNS/refused) are treated as unreachable; + transient errors are inconclusive to avoid false positives. + """ + result = { + "is_accessible": False, + "status_code": None, + "error": None, + "verdict": "inconclusive", + "check_method": "http", + } + + if not is_valid_url_format(url): + result["error"] = "Invalid URL format" + result["verdict"] = HARD_FAILURE + return result + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + } + + status_code = None + error = None + for attempt in range(3): + try: + response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) + status_code = response.status_code + error = None + if status_code == 429 and attempt < 2: + time.sleep(1 + attempt) + continue + break + except requests.exceptions.RequestException as e: + status_code = None + error = str(e) + if attempt < 2: + time.sleep(0.5) + continue + break + except Exception as e: + status_code = None + error = f"Unexpected error: {str(e)}" + break + + result["status_code"] = status_code + result["error"] = error + result["verdict"] = classify_status(status_code, error) + result["is_accessible"] = result["verdict"] == "accessible" + return result + + +def check_ci_url_status(url: str, timeout: int = 10) -> Dict: + """ + Check a CI URL. GitHub Actions links go through the REST API (HTML always + answers 200 for /actions/...); everything else falls back to HTTP. + """ + if parse_github_actions_url(url) is not None: + return check_github_ci_status(url, timeout) + return _check_ci_url_status_html(url, timeout) + + +def _is_unreachable(url_status: Dict) -> bool: + """True if a CI URL is definitively unreachable (hard failure).""" + verdict = url_status.get("verdict") + if verdict: + return verdict == HARD_FAILURE + if url_status.get("status_code") in (404, 410): + return True + error = url_status.get("error") + return isinstance(error, str) and "Invalid URL format" in error + + +def _codemeta_ci_urls(somef_data: Dict): + """CI URLs declared in codemeta.json (contIntegration / continuousIntegration). + + SoMEF does not always surface the codemeta continuous-integration field under + the ``continuous_integration`` category, so fall back to reading the + codemeta.json file directly. A list value is flattened. + """ + urls = [] + entries = somef_data.get("continuous_integration") + if isinstance(entries, list): + for entry in entries: + source = str(entry.get("source", "")) + technique = entry.get("technique", "") + if "codemeta.json" in source.lower() or ( + technique == "code_parser" and "codemeta" in source.lower()): + value = (entry.get("result") or {}).get("value") + if isinstance(value, str): + urls.append(value) + elif isinstance(value, list): + urls.extend(v for v in value if isinstance(v, str)) + + if urls: + return urls + + codemeta_url = find_source_url(somef_data, "codemeta.json", codemeta_root_only=True) + if codemeta_url: + text = fetch_text(codemeta_url) + if text: + try: + data = json.loads(text) + except (json.JSONDecodeError, ValueError): + data = {} + for key in ("continuousIntegration", "contIntegration", "codemeta:contIntegration"): + value = data.get(key) + if isinstance(value, str) and value.startswith(("http://", "https://")): + urls.append(value) + elif isinstance(value, dict) and isinstance(value.get("url"), str): + urls.append(value["url"]) + elif isinstance(value, list): + urls.extend(v for v in value if isinstance(v, str)) + + return urls + + +def detect_ci_404_pitfall(somef_data: Dict, file_name: str) -> Dict: + """ + Detect when codemeta.json continuous integration link points to a dead end. + """ + result = { + "has_pitfall": False, + "file_name": file_name, + "ci_url": None, + "source": None, + "status_code": None, + "error": None, + "check_method": None, + "api_status": None, + "workflow_state": None, + } + + for ci_url in _codemeta_ci_urls(somef_data): + url_status = check_ci_url_status(ci_url) + + if _is_unreachable(url_status): + result["has_pitfall"] = True + result["ci_url"] = ci_url + result["source"] = "codemeta.json" + result["status_code"] = url_status.get("status_code") + result["error"] = url_status.get("error") + result["check_method"] = url_status.get("check_method") + result["api_status"] = url_status.get("api_status") + result["workflow_state"] = url_status.get("workflow_state") + break + + return result diff --git a/src/rsmetacheck/scripts/pitfalls/p016.py b/src/rsmetacheck/scripts/pitfalls/p016.py index fb5596c..73bb999 100644 --- a/src/rsmetacheck/scripts/pitfalls/p016.py +++ b/src/rsmetacheck/scripts/pitfalls/p016.py @@ -1,5 +1,6 @@ from typing import Dict import re +from rsmetacheck.utils.fetch_utils import is_root_codemeta_url, is_vendored_source def normalize_repository_url(url: str) -> str: """ @@ -55,8 +56,17 @@ def detect_different_repository_pitfall(somef_data: Dict, file_name: str) -> Dic repo_url = entry["result"]["value"] if technique == "GitHub_API": + if isinstance(repo_url, str) and re.search(r'/commit/[0-9a-f]{7,40}$', repo_url): + continue github_api_url = repo_url elif any(src in source.lower() for src in ["codemeta.json"]): + # Only consider the root-level codemeta.json. Nested ones are + # not the project's metadata and cause false positives. + source_str = str(source) + if not is_root_codemeta_url(source_str): + continue + if is_vendored_source(source_str): + continue metadata_urls.append({ "url": repo_url, "source": source, diff --git a/src/rsmetacheck/scripts/pitfalls/p017.py b/src/rsmetacheck/scripts/pitfalls/p017.py index dbb0b84..059fd2e 100644 --- a/src/rsmetacheck/scripts/pitfalls/p017.py +++ b/src/rsmetacheck/scripts/pitfalls/p017.py @@ -70,7 +70,15 @@ def get_other_metadata_versions(somef_data: Dict) -> list: if "codemeta.json" in source: continue - if technique == "code_parser" or any(src in source.lower() for src in metadata_sources): + # Only versions declared in a known package/metadata manifest count as + # "other" versions. Accepting any code_parser entry let unrelated + # parsers feed this comparison - notably SoMEF >= 0.11.5's .zenodo.json, + # whose deposit version legitimately differs from the package version. + # Entries with no provenance at all cannot be filtered by file name and + # are still accepted (source matching is case-insensitive: manifest + # names in the list are not all lower-case, e.g. "DESCRIPTION"). + has_known_source = any(src.lower() in source.lower() for src in metadata_sources) + if has_known_source or (technique == "code_parser" and not source.strip()): if "result" in entry and "value" in entry["result"]: other_versions.append({ "version": entry["result"]["value"], diff --git a/src/rsmetacheck/scripts/pitfalls/p018.py b/src/rsmetacheck/scripts/pitfalls/p018.py index 1b13206..d8236cc 100644 --- a/src/rsmetacheck/scripts/pitfalls/p018.py +++ b/src/rsmetacheck/scripts/pitfalls/p018.py @@ -51,11 +51,18 @@ def detect_raw_swhid_pitfall(somef_data: Dict, file_name: str) -> Dict: if "result" in entry and "value" in entry["result"]: identifier_value = entry["result"]["value"] - if is_raw_swhid(identifier_value): - result["has_pitfall"] = True - result["identifier_value"] = identifier_value - result["source"] = source - result["is_raw_swhid"] = True + # SoMEF < 0.11.5 kept an identifier array as a single list value; + # >= 0.11.5 emits one entry per element. Accept both shapes. + candidates = identifier_value if isinstance(identifier_value, list) else [identifier_value] + + for candidate in candidates: + if is_raw_swhid(candidate): + result["has_pitfall"] = True + result["identifier_value"] = candidate + result["source"] = source + result["is_raw_swhid"] = True + break + if result["has_pitfall"]: break return result \ No newline at end of file diff --git a/src/rsmetacheck/scripts/pitfalls/p019.py b/src/rsmetacheck/scripts/pitfalls/p019.py index 2a4c365..f3ea4de 100644 --- a/src/rsmetacheck/scripts/pitfalls/p019.py +++ b/src/rsmetacheck/scripts/pitfalls/p019.py @@ -1,11 +1,31 @@ +import json +import re from typing import Dict, List, Tuple + from rsmetacheck.utils.pitfall_utils import extract_metadata_source_filename +from rsmetacheck.utils.fetch_utils import find_source_url, fetch_text + +# Only these files are metadata sources for P019 (AUTHORS/Dockerfile/etc. are not). +METADATA_AUTHOR_SOURCES = { + "codemeta.json", "description", "setup.py", "setup.cfg", "package.json", + "pyproject.toml", "pom.xml", "requirements.txt", "citation.cff", + "cargo.toml", "composer.json", "bower.json", +} + +# Deposit records are not package manifests. SoMEF >= 0.11.5 parses a root +# .zenodo.json and emits its authors with technique code_parser; counting them +# here would report a false author-count inconsistency against the manifest. +EXCLUDED_AUTHOR_SOURCES = {"zenodo.json", ".zenodo.json", "zenodo_deposit.json"} def extract_authors_from_somef(somef_data: Dict) -> List[Dict[str, any]]: """ - Extract all author entries from different sources in SoMEF output. - Returns a list of dicts with source and author count information. + Extract author counts grouped by metadata source file. + + SoMEF 0.11.2 emits one entry per author (each with a dict result), so the + author count for a given source file is the number of entries pointing at + that file. Aggregating by source file (rather than counting per entry) + makes "codemeta.json has 37 authors vs setup.py has 1" detectable. """ if "author" not in somef_data: return [] @@ -14,7 +34,8 @@ def extract_authors_from_somef(somef_data: Dict) -> List[Dict[str, any]]: if not isinstance(author_entries, list): return [] - results = [] + grouped = {} + order = [] for entry in author_entries: if "source" not in entry or "result" not in entry: @@ -23,28 +44,29 @@ def extract_authors_from_somef(somef_data: Dict) -> List[Dict[str, any]]: source = entry["source"] result = entry["result"] - author_count = 0 - authors_list = [] - if isinstance(result, list): - author_count = len(result) - authors_list = [get_author_identifier(author) for author in result] - elif isinstance(result, dict): - author_count = 1 - authors_list = [get_author_identifier(result)] - elif isinstance(result, str): - author_count = 1 - authors_list = [get_author_identifier(result)] - - if author_count > 0: - results.append({ + if not result: + continue + identifiers = [get_author_identifier(author) for author in result] + elif isinstance(result, (dict, str)): + identifiers = [get_author_identifier(result)] + else: + continue + + source_file = extract_metadata_source_filename(source) + if source_file in grouped: + grouped[source_file]["authors"].extend(identifiers) + grouped[source_file]["author_count"] += len(identifiers) + else: + grouped[source_file] = { "source": source, - "source_file": extract_metadata_source_filename(source), - "author_count": author_count, - "authors": authors_list - }) + "source_file": source_file, + "author_count": len(identifiers), + "authors": identifiers, + } + order.append(source_file) - return results + return [grouped[source_file] for source_file in order] def get_author_identifier(author: any) -> str: @@ -66,6 +88,60 @@ def get_author_identifier(author: any) -> str: return str(author) +def _fetch_setup_py_authors(somef_data: Dict) -> Tuple[int, List[str]]: + """Read the author(s) from setup.py (SoMEF often misses setuptools.setup).""" + url = find_source_url(somef_data, "setup.py") + if not url: + return 0, [] + text = fetch_text(url) + if not text: + return 0, [] + match = re.search(r'(?:^|\n)\s*author\s*=\s*["\']([^"\']+)["\']', text) + if not match: + return 0, [] + value = match.group(1) + parts = [p.strip() for p in re.split(r'\s*(?:,|;|\band\b|\s&\s)\s*', value) if p.strip()] + return len(parts), parts + + +def _fetch_description_authors(somef_data: Dict) -> Tuple[int, List[str]]: + """Read Authors@R from an R DESCRIPTION file.""" + url = find_source_url(somef_data, "DESCRIPTION") + if not url: + return 0, [] + text = fetch_text(url) + if not text: + return 0, [] + names = re.findall(r'person\s*\(\s*["\']([^"\']+)["\']', text) + count = len(re.findall(r'\bperson\s*\(', text)) + if count == 0: + return 0, [] + return count, names + + +def _fetch_package_json_authors(somef_data: Dict) -> Tuple[int, List[str]]: + """Read the author(s) from package.json.""" + url = find_source_url(somef_data, "package.json") + if not url: + return 0, [] + text = fetch_text(url) + if not text: + return 0, [] + try: + data = json.loads(text) + except (json.JSONDecodeError, ValueError): + return 0, [] + author = data.get("author") or data.get("authors") + if isinstance(author, list): + authors = [(x.get("name") if isinstance(x, dict) else str(x)) for x in author] + return len(authors), authors + if isinstance(author, dict): + return 1, [str(author.get("name", ""))] + if isinstance(author, str): + return 1, [author] + return 0, [] + + def find_author_count_inconsistencies(author_sources: List[Dict]) -> Tuple[bool, List[Dict]]: """ Check if there are inconsistencies in author counts across different sources. @@ -124,7 +200,29 @@ def detect_inconsistent_author_count(somef_data: Dict, file_name: str) -> Dict: "max_author_count": 0 } - author_sources = extract_authors_from_somef(somef_data) + author_sources = [ + src for src in extract_authors_from_somef(somef_data) + if src["source_file"].lower() in METADATA_AUTHOR_SOURCES + and src["source_file"].lower() not in EXCLUDED_AUTHOR_SOURCES + ] + + # SoMEF 0.11.2 often does not extract setup.py/package.json authors; read + # those files directly when they are present in the repository. + existing_files = {s["source_file"].lower() for s in author_sources} + for src_file, fetcher in (("setup.py", _fetch_setup_py_authors), + ("package.json", _fetch_package_json_authors), + ("description", _fetch_description_authors)): + if src_file in existing_files: + continue + count, authors = fetcher(somef_data) + if count and count > 0: + author_sources.append({ + "source": src_file, + "source_file": src_file, + "author_count": count, + "authors": authors, + }) + existing_files.add(src_file) if not author_sources: return result @@ -142,4 +240,4 @@ def detect_inconsistent_author_count(somef_data: Dict, file_name: str) -> Dict: result["has_warning"] = True result["inconsistencies"] = inconsistencies - return result \ No newline at end of file + return result diff --git a/src/rsmetacheck/scripts/warnings/w005.py b/src/rsmetacheck/scripts/warnings/w005.py index 22ebda8..92fcc02 100644 --- a/src/rsmetacheck/scripts/warnings/w005.py +++ b/src/rsmetacheck/scripts/warnings/w005.py @@ -45,6 +45,15 @@ def detect_multiple_requirements_in_string(requirement_string: str) -> List[str] # Clean the string req_str = requirement_string.strip() + # Not a package list: Software Heritage identifiers, URLs/VCS refs, and + # natural-language prose ("at least one of ... e.g. NEURON, NEST, ..."). + if re.search(r'\bswh:1:', req_str): + return [] + if req_str.startswith(('http://', 'https://', 'git+', 'git://')): + return [] + if re.search(r'\b(?:e\.g\.|i\.e\.|such as|at least one|for example)\b', req_str, re.IGNORECASE): + return [] + # Patterns that might indicate multiple requirements if bool(re.search(r'\b(npm|bash|cd|pip|install)\b', req_str.lower())): return [] diff --git a/src/rsmetacheck/scripts/warnings/w006.py b/src/rsmetacheck/scripts/warnings/w006.py index 49b4f3c..8f15179 100644 --- a/src/rsmetacheck/scripts/warnings/w006.py +++ b/src/rsmetacheck/scripts/warnings/w006.py @@ -1,168 +1,220 @@ -from typing import Dict, List -import re - -from rsmetacheck.utils.pitfall_utils import extract_metadata_source_filename - - -def is_valid_identifier(identifier: str) -> bool: - """ - Check if identifier appears to be a valid unique identifier (DOI, URL) rather than a name. - """ - if not identifier or not isinstance(identifier, str): - return False - - identifier = identifier.strip() - - if not identifier: - return False - - # Check for DOI patterns - doi_patterns = [ - r'^doi:10\.\d+/.+', # doi:10.xxxx/yyyy format - r'^10\.\d+/.+' # 10.xxxx/yyyy format (bare DOI) - ] - - for pattern in doi_patterns: - if re.match(pattern, identifier, re.IGNORECASE): - return True - - if identifier.lower() in ['doi:', '10.']: - return False - - url_pattern = r'^https?://.+' - if re.match(url_pattern, identifier, re.IGNORECASE): - return True - - if identifier.lower().startswith('ftp://'): - return False - - if ' ' in identifier and not any(char in identifier for char in ['/', ':', '.']): - return False - - if identifier.replace(' ', '').replace('-', '').replace('_', '').isalpha(): - return False - - return True - - -def has_doi_in_other_sources(identifier_entries: List[Dict]) -> bool: - """ - Check if there's a valid DOI in non-codemeta sources. - """ - for entry in identifier_entries: - source = entry.get("source", "") - - if "codemeta.json" in source.lower(): - continue - - if "result" in entry and "value" in entry["result"]: - identifier_value = entry["result"]["value"] - - if isinstance(identifier_value, str): - # Check for DOI patterns - doi_patterns = [ - r'^doi:10\.\d+/.+', # doi:10.xxxx/yyyy format - r'^10\.\d+/.+' # 10.xxxx/yyyy format (bare DOI) - ] - - for pattern in doi_patterns: - if re.match(pattern, identifier_value, re.IGNORECASE): - return True - - return False - - -def detect_identifier_name_warning(somef_data: Dict, file_name: str) -> Dict: - """ - Detect when codemeta.json identifier is a name instead of a valid unique identifier, - but an identifier exists elsewhere. - """ - result = { - "has_warning": False, - "file_name": file_name, - "codemeta_identifier": None, - "other_identifier": None, - "codemeta_source": None, - "other_source": None, - "has_valid_identifier_elsewhere": False, - "other_identifiers": [] - } - - if "identifier" not in somef_data: - return result - - identifier_entries = somef_data["identifier"] - if not isinstance(identifier_entries, list): - return result - - codemeta_identifier = None - codemeta_source = None - codemeta_has_valid_id = False - other_identifier = None - other_source = None - other_identifiers = [] - - for entry in identifier_entries: - source = entry.get("source", "") - technique = entry.get("technique", "") - - is_codemeta = ( - "codemeta.json" in source.lower() or - (technique == "code_parser" and "codemeta" in source.lower()) - ) - - if is_codemeta: - if "result" in entry and "value" in entry["result"]: - if codemeta_identifier is None: - raw_value = entry["result"]["value"] - codemeta_source = source - if isinstance(raw_value, list): - codemeta_has_valid_id = any( - isinstance(item, str) and is_valid_identifier(item) - for item in raw_value - ) - codemeta_identifier = ", ".join(str(item) for item in raw_value) if raw_value else None - else: - codemeta_identifier = raw_value - break - - for entry in identifier_entries: - source = entry.get("source", "") - technique = entry.get("technique", "") - - is_codemeta = ( - "codemeta.json" in source.lower() or - (technique == "code_parser" and "codemeta" in source.lower()) - ) - - if is_codemeta: - continue - - if "result" in entry and "value" in entry["result"]: - identifier_value = entry["result"]["value"] - - if is_valid_identifier(identifier_value): - other_identifiers.append({ - "value": identifier_value, - "source": source - }) - - if other_identifier is None: - other_identifier = identifier_value - other_source = source - - result["codemeta_identifier"] = codemeta_identifier - result["codemeta_source"] = codemeta_source - result["metadata_source_file"] = extract_metadata_source_filename(codemeta_source) if codemeta_source else "codemeta.json" - result["other_identifier"] = other_identifier - result["other_source"] = other_source - result["other_identifiers"] = other_identifiers - result["has_valid_identifier_elsewhere"] = len(other_identifiers) > 0 - - if (codemeta_identifier and - not codemeta_has_valid_id and - not is_valid_identifier(codemeta_identifier) and - other_identifier): - result["has_warning"] = True - +from typing import Dict, List +import re + +from rsmetacheck.utils.pitfall_utils import extract_metadata_source_filename +from rsmetacheck.utils.fetch_utils import fetch_text + + +def is_valid_identifier(identifier: str) -> bool: + """ + Check if identifier appears to be a valid unique identifier (DOI, URL) rather than a name. + """ + if not identifier or not isinstance(identifier, str): + return False + + identifier = identifier.strip() + + if not identifier: + return False + + # Check for DOI patterns + doi_patterns = [ + r'^doi:10\.\d+/.+', # doi:10.xxxx/yyyy format + r'^10\.\d+/.+' # 10.xxxx/yyyy format (bare DOI) + ] + + for pattern in doi_patterns: + if re.match(pattern, identifier, re.IGNORECASE): + return True + + if identifier.lower() in ['doi:', '10.']: + return False + + url_pattern = r'^https?://.+' + if re.match(url_pattern, identifier, re.IGNORECASE): + return True + + if identifier.lower().startswith('ftp://'): + return False + + if ' ' in identifier and not any(char in identifier for char in ['/', ':', '.']): + return False + + if identifier.replace(' ', '').replace('-', '').replace('_', '').isalpha(): + return False + + return True + + +def _is_strict_identifier(value) -> bool: + """True only for an actual URL or DOI (not citation prose).""" + if not value or not isinstance(value, str): + return False + v = value.strip() + if re.match(r'^https?://\S+$', v, re.IGNORECASE): + return True + if re.match(r'^(doi:)?10\.\d{4,9}/\S+$', v, re.IGNORECASE): + return True + return False + + +def has_doi_in_other_sources(identifier_entries: List[Dict]) -> bool: + """ + Check if there's a valid DOI in non-codemeta sources. + """ + for entry in identifier_entries: + source = entry.get("source", "") + + if "codemeta.json" in source.lower(): + continue + + if "result" in entry and "value" in entry["result"]: + identifier_value = entry["result"]["value"] + + if isinstance(identifier_value, str): + # Check for DOI patterns + doi_patterns = [ + r'^doi:10\.\d+/.+', # doi:10.xxxx/yyyy format + r'^10\.\d+/.+' # 10.xxxx/yyyy format (bare DOI) + ] + + for pattern in doi_patterns: + if re.match(pattern, identifier_value, re.IGNORECASE): + return True + + return False + + +def detect_identifier_name_warning(somef_data: Dict, file_name: str) -> Dict: + """ + Detect when codemeta.json identifier is a name instead of a valid unique identifier, + but an identifier exists elsewhere. + """ + result = { + "has_warning": False, + "file_name": file_name, + "codemeta_identifier": None, + "other_identifier": None, + "codemeta_source": None, + "other_source": None, + "has_valid_identifier_elsewhere": False, + "other_identifiers": [] + } + + if "identifier" not in somef_data: + return result + + identifier_entries = somef_data["identifier"] + if not isinstance(identifier_entries, list): + return result + + codemeta_identifier = None + codemeta_source = None + codemeta_has_valid_id = False + other_identifier = None + other_source = None + other_identifiers = [] + + for entry in identifier_entries: + source = entry.get("source", "") + technique = entry.get("technique", "") + + is_codemeta = ( + "codemeta.json" in source.lower() or + (technique == "code_parser" and "codemeta" in source.lower()) + ) + + if is_codemeta: + if "result" in entry and "value" in entry["result"]: + # SoMEF < 0.11.5 kept an identifier array as one list value; + # >= 0.11.5 emits one entry per element, so validity and the + # reported value cannot be decided from the first entry alone. + # Warn only when NO codemeta identifier is a real unique id, + # and report the first one that is a bare name. + raw_value = entry["result"]["value"] + items = raw_value if isinstance(raw_value, list) else [raw_value] + for item in items: + if not isinstance(item, str): + continue + if is_valid_identifier(item): + codemeta_has_valid_id = True + elif codemeta_identifier is None: + codemeta_identifier = item + codemeta_source = source + if codemeta_source is None: + codemeta_source = source + + for entry in identifier_entries: + source = entry.get("source", "") + technique = entry.get("technique", "") + + is_codemeta = ( + "codemeta.json" in source.lower() or + (technique == "code_parser" and "codemeta" in source.lower()) + ) + + if is_codemeta: + continue + + if "result" in entry and "value" in entry["result"]: + identifier_value = entry["result"]["value"] + + if is_valid_identifier(identifier_value): + other_identifiers.append({ + "value": identifier_value, + "source": source + }) + + if other_identifier is None: + other_identifier = identifier_value + other_source = source + + # SoMEF 0.11.2 stores citation DOIs under `citation`, not `identifier`. + if not other_identifier: + for entry in somef_data.get("citation") or []: + result_data = entry.get("result") or {} + for candidate in (result_data.get("value"), result_data.get("url"), result_data.get("doi")): + if _is_strict_identifier(candidate): + other_identifiers.append({"value": candidate, "source": entry.get("source", "citation")}) + other_identifier = candidate + other_source = entry.get("source", "citation") + break + if other_identifier: + break + + # Fallback: look for a DOI/Zenodo badge in the README itself. + if not other_identifier: + readme_entries = somef_data.get("readme_url") + if isinstance(readme_entries, list): + for entry in readme_entries: + url = (entry.get("result") or {}).get("value") + if not isinstance(url, str): + continue + text = fetch_text(url) + if not text: + continue + match = re.search( + r'10\.\d{4,9}/[^\s"\'<>)\]]+|zenodo\.org/badge/[^\s"\'<>)\]]+', + text, re.IGNORECASE) + if match: + value = match.group(0) + other_identifiers.append({"value": value, "source": url}) + other_identifier = value + other_source = url + break + + result["codemeta_identifier"] = codemeta_identifier + result["codemeta_source"] = codemeta_source + result["metadata_source_file"] = extract_metadata_source_filename(codemeta_source) if codemeta_source else "codemeta.json" + result["other_identifier"] = other_identifier + result["other_source"] = other_source + result["other_identifiers"] = other_identifiers + result["has_valid_identifier_elsewhere"] = len(other_identifiers) > 0 + + if (codemeta_identifier and + not codemeta_has_valid_id and + not is_valid_identifier(codemeta_identifier) and + other_identifier): + result["has_warning"] = True + return result \ No newline at end of file diff --git a/src/rsmetacheck/utils/fetch_utils.py b/src/rsmetacheck/utils/fetch_utils.py new file mode 100644 index 0000000..dbabaf0 --- /dev/null +++ b/src/rsmetacheck/utils/fetch_utils.py @@ -0,0 +1,207 @@ +"""Shared HTTP helpers for rsmetacheck detectors. + +Kept dependency-light (only `requests`) so individual detectors can be tested +with the standard `@patch('...pXXX.requests.get')` approach. +""" +import re +import time +from typing import Any, Dict, Optional, Tuple +from urllib.parse import urlparse + +import requests + +DEFAULT_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0 Safari/537.36" + ), +} + +# --- pure classification --------------------------------------------------- + +ACCESSIBLE = "accessible" +HARD_FAILURE = "hard_failure" +INCONCLUSIVE = "inconclusive" + + +def classify_status(status_code: Optional[int], error: Optional[str]) -> str: + """Classify an HTTP result into accessible / hard_failure / inconclusive. + + Only definitive failures are treated as hard failures (404/410 and DNS / + connection-refused errors). Transient conditions (rate limiting, server + errors, timeouts, SSL issues) are inconclusive so detectors do not emit + false positives when the network flakes. + """ + if status_code is not None: + if 200 <= status_code < 400 and status_code != 300: + return ACCESSIBLE + if status_code in (404, 410): + return HARD_FAILURE + return INCONCLUSIVE + if error: + low = error.lower() + if any(token in low for token in ( + "gaierror", "name or service not known", "getaddrinfo", + "connectionrefusederror", "errno 111", "connection refused")): + return HARD_FAILURE + return INCONCLUSIVE + return INCONCLUSIVE + + +def is_hard_failure(status_code: Optional[int], error: Optional[str]) -> bool: + return classify_status(status_code, error) == HARD_FAILURE + + +# --- network text fetching (cached) --------------------------------------- + +_cache: Dict[str, Optional[str]] = {} + + +def fetch_text(url: str, timeout: int = 20, retries: int = 1) -> Optional[str]: + """Fetch a text file over HTTP(S), with an in-memory cache. + + Used to read LICENSE / codemeta.json / pyproject.toml / README from the raw + source URL recorded in a SoMEF output, so detectors can inspect content that + SoMEF 0.11.2 no longer embeds in the JSON. + """ + if not url or not url.startswith(("http://", "https://")): + return None + if url in _cache: + return _cache[url] + + text: Optional[str] = None + for attempt in range(retries + 1): + try: + response = requests.get(url, timeout=timeout, headers=DEFAULT_HEADERS, + allow_redirects=True) + if response.status_code == 200: + text = response.text + break + if response.status_code == 429 and attempt < retries: + time.sleep(1 + attempt) + continue + break + except requests.exceptions.RequestException: + if attempt < retries: + time.sleep(0.5) + continue + break + + _cache[url] = text + return text + + +# --- source URL helpers ---------------------------------------------------- + +def _as_strings(value: Any): + if isinstance(value, (list, tuple)): + return [str(x) for x in value] + return [str(value)] + + +def is_root_codemeta_url(source: str) -> bool: + """True if `source` looks like a repository-level (non-nested) codemeta.json. + + A root codemeta source is either a relative path (e.g. ``codemeta.json`` or + ``repository/codemeta.json``) or a raw GitHub URL with at most + ``///codemeta.json``. Deeply nested or vendored codemetas + (``.../third_party/.../codemeta.json``, ``.../docs/joss/.../codemeta.json``) + have more path segments and are ignored by codemeta-based detectors. + """ + if not source: + return False + try: + parts = [seg for seg in urlparse(str(source)).path.split("/") if seg] + return parts[-1].lower() == "codemeta.json" and len(parts) <= 4 + except Exception: + return False + + +def is_vendored_source(source: str) -> bool: + """True if the source path points inside a vendored third-party directory.""" + if not source: + return False + low = str(source).lower() + return any(f"/{d}/" in low for d in ( + "third_party", "third-party", "externals", "external", "vendor", + "subprojects", "deps", "node_modules")) + + +def find_source_url(somef_data: Dict, target_filename: str, + codemeta_root_only: bool = False) -> Optional[str]: + """Return the first source URL whose path mentions `target_filename`.""" + target = target_filename.lower() + for value in somef_data.values(): + if not isinstance(value, list): + continue + for entry in value: + if not isinstance(entry, dict): + continue + source = entry.get("source") + if not source: + continue + for src in _as_strings(source): + if not src.startswith(("http://", "https://")): + continue + if target not in src.lower(): + continue + if codemeta_root_only and not is_root_codemeta_url(src): + continue + return src + return None + + +_RAW_GITHUB_RE = re.compile(r'^https?://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/') + +LICENSE_FILENAMES = [ + "LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "LICENCE.md", + "LICENCE.txt", "COPYING", "COPYING.md", "COPYING.txt", "COPYRIGHT", + "LICENSE.rst", "COPYING.LESSER", +] + + +def find_repo_root_url(somef_data: Dict) -> Optional[str]: + """Derive the repository root URL at the analyzed ref from any SoMEF source. + + For a pinned run the ref is the recorded commit SHA, so probing files under + this root stays at the same revision (no drift). + """ + for value in somef_data.values(): + if not isinstance(value, list): + continue + for entry in value: + if not isinstance(entry, dict): + continue + source = entry.get("source") + if not source: + continue + for src in _as_strings(source): + if not src.startswith(("http://", "https://")): + continue + m = _RAW_GITHUB_RE.match(src) + if m: + return f"https://raw.githubusercontent.com/{m.group(1)}/{m.group(2)}/{m.group(3)}/" + if "/-/raw/" in src: + pre, rest = src.split("/-/raw/", 1) + ref = rest.split("/", 1)[0] + if ref: + return f"{pre}/-/raw/{ref}/" + return None + + +def fetch_license_file(somef_data: Dict) -> Tuple[Optional[str], Optional[str]]: + """Probe the repository root for a license file and return (url, text). + + Used when SoMEF merged/normalized the license entries and dropped the + LICENSE/COPYING file source (so no entry points at the file itself). + Returns (None, None) when nothing is found. + """ + root = find_repo_root_url(somef_data) + if not root: + return None, None + for filename in LICENSE_FILENAMES: + url = root + filename + text = fetch_text(url) + if text and text.strip(): + return url, text + return None, None diff --git a/tests/test_p002_family.py b/tests/test_p002_family.py new file mode 100644 index 0000000..f1a56b5 --- /dev/null +++ b/tests/test_p002_family.py @@ -0,0 +1,220 @@ +"""Family-aware P002 tests. + +GPL/LGPL/AGPL/Apache/MPL/EPL/CDDL ship an appendix of instructions whose +placeholder tokens are part of the licence; those must never be reported. +Permissive licences (MIT/MIT-0/BSD/0BSD/ISC/X11/zlib/Unlicense) are checked in +the copyright header only. +""" +import pytest + +from rsmetacheck.scripts.pitfalls import p002 + +GPL3 = """GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License. +""" + +APACHE2 = """Apache License +Version 2.0, January 2004 + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +APPENDIX: How to apply the Apache License to your work. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0. +""" + +MIT_TEMPLATE = """MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"). +""" + +MIT_FILLED = """MIT License + +Copyright (c) 2026 Jane Doe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"). +""" + +BSD3_TEMPLATE = """BSD 3-Clause License + +Copyright (c) , +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +""" + +ISC_TEMPLATE = """ISC License + +Copyright (c) 4-digit year, Company or Person's Name + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. +""" + +ZERO_BSD_TEMPLATE = """Zero-Clause BSD + +Copyright (C) YEAR by AUTHOR EMAIL + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. +""" + +X11_TEMPLATE = """X11 License + +Copyright (C) X Consortium + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"). + +Except as contained in this notice, the name of the X Consortium shall not be +used in advertising. +""" + +ZLIB_TEMPLATE = """zlib License + +Copyright (C) Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied warranty. + +The origin of this software must not be misrepresented. +""" + + +@pytest.mark.parametrize("content,family", [ + (GPL3, "gpl"), + (APACHE2, "apache"), + (MIT_TEMPLATE, "mit"), + (BSD3_TEMPLATE, "bsd"), + (ISC_TEMPLATE, "isc"), + (X11_TEMPLATE, "x11"), + (ZLIB_TEMPLATE, "zlib"), + ("This is free and unencumbered software released into the public domain.", "unlicense"), + ("Some bespoke terms.\nCopyright 2026 ACME.", ""), +]) +def test_family_detection(content, family): + assert p002.detect_license_family(content) == family + + +def test_agpl_and_lgpl_are_detected_before_gpl(): + assert p002.detect_license_family( + "GNU AFFERO GENERAL PUBLIC LICENSE\nVersion 3") == "agpl" + assert p002.detect_license_family( + "GNU LESSER GENERAL PUBLIC LICENSE\nVersion 3") == "lgpl" + + +@pytest.mark.parametrize("content", [GPL3, APACHE2]) +def test_appendix_families_never_flagged(content): + """The 34-false-positive class: GPL/Apache appendix boilerplate.""" + analysis = p002.analyze_license_placeholders(content) + assert analysis["has_pitfall"] is False + assert analysis["decision"] == "exempt_appendix_family" + assert analysis["placeholders"] == [] + + +@pytest.mark.parametrize("content,token_hint", [ + (MIT_TEMPLATE, ""), + (BSD3_TEMPLATE, ""), + (ISC_TEMPLATE, "4-digit year"), + (ZERO_BSD_TEMPLATE, "YEAR"), + (X11_TEMPLATE, ""), + (ZLIB_TEMPLATE, ""), +]) +def test_permissive_templates_are_flagged(content, token_hint): + analysis = p002.analyze_license_placeholders(content) + assert analysis["has_pitfall"] is True, analysis + assert analysis["decision"] == "placeholder_in_copyright_header" + assert any(token_hint.lower() in token.lower() for token in analysis["placeholders"]), analysis + + +def test_filled_mit_is_clean(): + analysis = p002.analyze_license_placeholders(MIT_FILLED) + assert analysis["has_pitfall"] is False + assert analysis["decision"] == "clean" + + +def test_zero_bsd_is_not_reported_as_isc(): + assert p002.detect_license_family(ZERO_BSD_TEMPLATE) == "0bsd" + assert p002.detect_license_family(ISC_TEMPLATE) == "isc" + + +def test_placeholder_in_licence_body_is_ignored(): + """Unknown licence, token appears only after the body grant starts.""" + content = """Custom Licence + +Copyright 2026 ACME + +The Software is provided as is, and you may use freely. +""" + analysis = p002.analyze_license_placeholders(content) + assert analysis["has_pitfall"] is False + + +def test_unknown_licence_placeholder_in_header_is_flagged(): + content = """Custom Licence + +Copyright (c) +All rights reserved. +""" + assert p002.analyze_license_placeholders(content)["has_pitfall"] is True + + +def test_empty_content_is_clean(): + assert p002.analyze_license_placeholders("")["has_pitfall"] is False + assert p002.check_license_template_placeholders(None) is False + + +def _somef(license_text, source="https://example.org/repo/LICENSE"): + return {"license": [{ + "source": source, + "technique": "file_exploration", + "result": {"value": license_text, "type": "File_dump"}, + }]} + + +def test_detector_exempts_gpl(monkeypatch): + monkeypatch.setattr(p002, "fetch_text", lambda url, **kw: None) + result = p002.detect_license_template_placeholders(_somef(GPL3), "test.json") + assert result["has_pitfall"] is False + assert result["license_family"] == "gpl" + assert result["decision"] == "exempt_appendix_family" + + +def test_detector_flags_mit_template(monkeypatch): + monkeypatch.setattr(p002, "fetch_text", lambda url, **kw: None) + result = p002.detect_license_template_placeholders(_somef(MIT_TEMPLATE), "test.json") + assert result["has_pitfall"] is True + assert result["license_family"] == "mit" + assert result["placeholders_found"] is True + assert "" in result["placeholders"] + + +def test_detector_result_structure(monkeypatch): + monkeypatch.setattr(p002, "fetch_text", lambda url, **kw: None) + result = p002.detect_license_template_placeholders(_somef(MIT_FILLED), "test.json") + for key in ("has_pitfall", "file_name", "license_source", "placeholders_found", + "license_family", "placeholders", "decision"): + assert key in result diff --git a/tests/test_p011.py b/tests/test_p011.py index db01f5a..87f696b 100644 --- a/tests/test_p011.py +++ b/tests/test_p011.py @@ -1,4 +1,5 @@ import pytest +from unittest.mock import patch from rsmetacheck.scripts.pitfalls.p011 import ( is_valid_issue_tracker_format, detect_issue_tracker_format_pitfall @@ -109,7 +110,9 @@ def test_valid_format_no_pitfall(self): "result": {"value": "https://github.com/user/repo/issues"} }] } - result = detect_issue_tracker_format_pitfall(somef_data, "test.json") + with patch('rsmetacheck.scripts.pitfalls.p011._is_unreachable', + return_value={"status_code": 200, "error": None, "verdict": "accessible"}): + result = detect_issue_tracker_format_pitfall(somef_data, "test.json") assert result["has_pitfall"] is False def test_invalid_format_has_pitfall(self): @@ -245,7 +248,9 @@ def test_various_valid_urls_not_detected(self): "result": {"value": url} }] } - result = detect_issue_tracker_format_pitfall(somef_data, "test.json") + with patch('rsmetacheck.scripts.pitfalls.p011._is_unreachable', + return_value={"status_code": 200, "error": None, "verdict": "accessible"}): + result = detect_issue_tracker_format_pitfall(somef_data, "test.json") assert result["has_pitfall"] is False, f"URL should be valid: {url}" def test_wrong_technique_still_matches_source(self): diff --git a/tests/test_p012.py b/tests/test_p012.py index a12feb4..f712523 100644 --- a/tests/test_p012.py +++ b/tests/test_p012.py @@ -31,6 +31,11 @@ class TestExtractVersionFromDownloadUrl: ("https://example.com/releases/3.8.0/package.tar.gz", "3.8.0"), ("https://example.com/downloads/v1.2.3/file.zip", "1.2.3"), + # GitHub refs/tags and releases/tag URL forms (real-world cases) + ("https://github.com/umr-amap/StormR/archive/refs/tags/v0.2.2.zip", "0.2.2"), + ("https://github.com/vrineau/Agatta/archive/refs/tags/v0.8.1.tar.gz", "0.8.1"), + ("https://github.com/heuermh/sea-eagle/releases/tag/sea-eagle-0.6", "0.6"), + # No version ("https://github.com/user/repo/archive/main.tar.gz", None), ("https://example.com/downloads/package.tar.gz", None), @@ -152,6 +157,19 @@ class TestGetLatestReleaseVersion: "2.0.0" ), + # Prefixed tag (name-version format, e.g. sea-eagle-0.10) + ( + { + "releases": [{ + "result": { + "tag": "sea-eagle-0.10", + "name": "sea-eagle 0.10" + } + }] + }, + "0.10" + ), + # Empty tag, use name ( { @@ -267,6 +285,29 @@ class TestDetectOutdatedDownloadUrlPitfall: None ), + # downloadURL matches a stale codemeta version but release is newer (refs/tags URL form) + ( + { + "download_url": [{ + "source": "repository/codemeta.json", + "technique": "code_parser", + "result": {"value": "https://github.com/user/repo/archive/refs/tags/v0.2.2.zip"} + }], + "version": [{ + "source": "repository/codemeta.json", + "technique": "code_parser", + "result": {"value": "0.2.2"} + }], + "releases": [{ + "result": {"tag": "v0.3.0"} + }] + }, + "test_repo.json", + True, + "0.2.2", + "0.3.0" + ), + # REMOVED: Outdated download URL test case with v2.0.0 # Download URL from non-codemeta source diff --git a/tests/test_p015.py b/tests/test_p015.py index 1a9b96c..d2319bc 100644 --- a/tests/test_p015.py +++ b/tests/test_p015.py @@ -351,7 +351,7 @@ def test_multiple_ci_sources_mixed(self): @pytest.mark.parametrize("status_code", [401, 403, 500, 502, 503]) def test_various_error_status_codes(self, status_code): - """Test that various error status codes trigger pitfall""" + """Test that transient/server errors are NOT flagged (inconclusive)""" somef_data = { "continuous_integration": [{ "source": "repository/codemeta.json", @@ -364,5 +364,6 @@ def test_various_error_status_codes(self, status_code): return_value={"is_accessible": False, "status_code": status_code, "error": None}): result = detect_ci_404_pitfall(somef_data, "test.json") - assert result["has_pitfall"] is True - assert result["status_code"] == status_code \ No newline at end of file + assert result["has_pitfall"] is False + assert result["status_code"] is None + assert result["ci_url"] is None \ No newline at end of file diff --git a/tests/test_p015_github_api.py b/tests/test_p015_github_api.py new file mode 100644 index 0000000..8ac3a33 --- /dev/null +++ b/tests/test_p015_github_api.py @@ -0,0 +1,182 @@ +"""Tests for the GitHub-API path of P015 (dead CI links). + +GitHub answers 200 for any /actions/... page because it is a client-side route, +so the detector resolves workflow existence through the REST API instead. +""" +from unittest.mock import patch + +import pytest + +from rsmetacheck.scripts.pitfalls import p015 + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self._payload = payload + + def json(self): + if self._payload is None: + raise ValueError("no json") + return self._payload + + +def router(routes, calls=None): + """requests.get replacement: maps a URL substring to a FakeResponse.""" + def _get(url, **kwargs): + if calls is not None: + calls.append((url, kwargs)) + for fragment, response in routes.items(): + if fragment in url: + return response + return FakeResponse(404) + return _get + + +def test_workflow_exists_is_accessible(): + calls = [] + get = router({"/repos/o/r/actions/workflows/ci.yml": FakeResponse(200, {"state": "active"})}, calls) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://github.com/o/r/actions/workflows/ci.yml") + assert result["is_accessible"] is True + assert result["check_method"] == "github_api" + assert result["workflow_state"] == "active" + assert any("api.github.com" in url for url, _ in calls) + + +def test_missing_workflow_in_readable_repo_is_hard_failure(): + calls = [] + get = router({ + "/repos/o/r/actions/workflows/nightly.yml": FakeResponse(404), + "/repos/o/r": FakeResponse(200, {"full_name": "o/r"}), + }, calls) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status( + "https://github.com/o/r/actions/workflows/nightly.yml") + assert result["verdict"] == p015.HARD_FAILURE + assert result["status_code"] == 404 + assert "does not exist" in result["error"] + # repo probe must use the bare repo endpoint, not the actions one + assert any(url.endswith("/repos/o/r") for url, _ in calls) + + +def test_missing_workflow_in_unreadable_repo_is_inconclusive(): + get = router({ + "/repos/o/r/actions/workflows/ci.yml": FakeResponse(404), + "/repos/o/r": FakeResponse(404), + }) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://github.com/o/r/actions/workflows/ci.yml") + assert result["verdict"] == "inconclusive" + assert result["is_accessible"] is False + + +def test_badge_url_uses_api_not_image(): + get = router({"/actions/workflows/ci.yml": FakeResponse(200, {"state": "active"})}) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status( + "https://github.com/o/r/workflows/ci.yml/badge.svg?branch=main") + assert result["is_accessible"] is True + assert result["check_method"] == "github_api" + + +def test_blob_url_missing_file_hard_failure(): + get = router({ + "/contents/.github/workflows/ci.yml": FakeResponse(404), + "/repos/o/r": FakeResponse(200, {}), + }) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status( + "https://github.com/o/r/blob/main/.github/workflows/ci.yml") + assert result["verdict"] == p015.HARD_FAILURE + + +def test_actions_list_with_no_workflows_is_hard_failure(): + get = router({"/actions/workflows": FakeResponse(200, {"total_count": 0, "workflows": []})}) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://github.com/o/r/actions") + assert result["verdict"] == p015.HARD_FAILURE + assert "no workflows" in result["error"] + + +def test_rate_limit_is_inconclusive(): + get = router({"/actions/workflows/ci.yml": FakeResponse(403)}) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://github.com/o/r/actions/workflows/ci.yml") + assert result["verdict"] == "inconclusive" + assert "rate limited" in result["error"].lower() + + +def test_non_actions_github_url_falls_back_to_html(): + get = router({"github.com/o/r/releases": FakeResponse(200)}) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://github.com/o/r/releases") + assert result["check_method"] == "http" + + +def test_non_github_url_uses_html(): + get = router({"travis-ci.org/o/r": FakeResponse(404)}) + with patch.object(p015.requests, "get", get): + result = p015.check_ci_url_status("https://travis-ci.org/o/r") + assert result["check_method"] == "http" + assert result["verdict"] == p015.HARD_FAILURE + + +def test_network_error_is_inconclusive(): + def boom(url, **kwargs): + raise p015.requests.exceptions.ConnectionError("boom") + + with patch.object(p015.requests, "get", boom): + result = p015.check_ci_url_status("https://github.com/o/r/actions/workflows/ci.yml") + assert result["verdict"] == "inconclusive" + + +def test_token_header_sent_when_configured(monkeypatch): + calls = [] + get = router({"/actions/workflows": FakeResponse(200, {"total_count": 1})}, calls) + monkeypatch.setenv("GITHUB_API_TOKEN", "dummy-token") + with patch.object(p015.requests, "get", get): + p015.check_ci_url_status("https://github.com/o/r/actions") + assert calls[0][1]["headers"]["Authorization"] == "token dummy-token" + + +def test_no_token_header_when_unset(monkeypatch): + calls = [] + get = router({"/actions/workflows": FakeResponse(200, {"total_count": 1})}, calls) + for var in ("GITHUB_API_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"): + monkeypatch.delenv(var, raising=False) + with patch.object(p015.requests, "get", get): + p015.check_ci_url_status("https://github.com/o/r/actions") + assert "Authorization" not in calls[0][1]["headers"] + + +@pytest.mark.parametrize("url,expected_kind", [ + ("https://github.com/o/r/actions/workflows/ci.yml", "workflow_file"), + ("https://github.com/o/r/actions", "actions_list"), + ("https://github.com/o/r/actions/workflows", "actions_list"), + ("https://github.com/o/r/workflows/ci.yml/badge.svg", "workflow_file"), + ("https://github.com/o/r/blob/main/.github/workflows/ci.yml", "workflow_blob"), + ("https://github.com/o/r/tree/main/.github/workflows", "tree_workflows"), + ("https://github.com/o/r/releases", None), + ("https://gitlab.com/o/r/-/pipelines", None), +]) +def test_url_parsing(url, expected_kind): + target = p015.parse_github_actions_url(url) + assert (target or {}).get("kind") == expected_kind + + +def test_detector_reports_api_method(): + somef_data = {"continuous_integration": [{ + "source": "repository/codemeta.json", + "technique": "code_parser", + "result": {"value": "https://github.com/o/r/actions/workflows/ci.yml"}, + }]} + get = router({ + "/actions/workflows/ci.yml": FakeResponse(404), + "/repos/o/r": FakeResponse(200, {}), + }) + with patch.object(p015.requests, "get", get): + result = p015.detect_ci_404_pitfall(somef_data, "test.json") + assert result["has_pitfall"] is True + assert result["check_method"] == "github_api" + assert result["api_status"] == 404