From 49bd922d1db99ee152e3c1428317aa07a52e21eb Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 28 Jan 2022 12:42:18 +0800 Subject: [PATCH 1/4] Merge pull request #128 from ligangty/main Fix: tests failure due to new boto3 release --- tests/base.py | 2 ++ tests/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index 88eeeba8..011a5b0f 100644 --- a/tests/base.py +++ b/tests/base.py @@ -26,6 +26,7 @@ from charon.storage import PRODUCT_META_KEY, CHECKSUM_META_KEY from tests.commons import TEST_BUCKET from boto3_type_annotations import s3 +from moto import mock_s3 SHORT_TEST_PREFIX = "ga" @@ -82,6 +83,7 @@ def get_config_base(self) -> str: return os.path.join(self.get_temp_dir(), '.charon') +@mock_s3 class PackageBaseTest(BaseTest): def setUp(self): super().setUp() diff --git a/tests/requirements.txt b/tests/requirements.txt index 55d4ddd3..de68f7b1 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,5 +5,5 @@ pytest-cov pytest-html flake8 requests-mock -moto +moto==3.0.2.dev12 From a3c66da01ea88d79f3df5f290294328fe8d18215 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Tue, 8 Feb 2022 22:44:03 +0800 Subject: [PATCH 2/4] Merge pull request #130 from ligangty/main Chore: Update year in license header --- charon/__init__.py | 2 +- charon/cmd/__init__.py | 2 +- charon/cmd/command.py | 2 +- charon/config.py | 2 +- charon/constants.py | 2 +- charon/pkgs/__init__.py | 2 +- charon/pkgs/indexing.py | 2 +- charon/pkgs/maven.py | 2 +- charon/pkgs/npm.py | 2 +- charon/storage.py | 2 +- charon/utils/__init__.py | 2 +- charon/utils/archive.py | 2 +- charon/utils/files.py | 2 +- charon/utils/logs.py | 2 +- setup.py | 2 +- test.sh | 3 +++ tests/base.py | 2 +- tests/test_config.py | 2 +- tests/test_indexing.py | 2 +- tests/test_maven_del.py | 2 +- tests/test_maven_index.py | 2 +- tests/test_maven_meta.py | 2 +- tests/test_maven_upload.py | 2 +- tests/test_npm_del.py | 2 +- tests/test_npm_index.py | 2 +- tests/test_npm_meta.py | 2 +- tests/test_npm_upload.py | 2 +- tests/test_pkgs_dryrun.py | 2 +- tests/test_s3client.py | 2 +- tests/test_util.py | 2 +- 30 files changed, 32 insertions(+), 29 deletions(-) diff --git a/charon/__init__.py b/charon/__init__.py index eb8203f5..f0a1e4d2 100644 --- a/charon/__init__.py +++ b/charon/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index dbff095e..9eefcae0 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/cmd/command.py b/charon/cmd/command.py index fbc26bd3..6d2f556a 100644 --- a/charon/cmd/command.py +++ b/charon/cmd/command.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/config.py b/charon/config.py index 3e464fb8..00b1398d 100644 --- a/charon/config.py +++ b/charon/config.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/constants.py b/charon/constants.py index 242030f3..d38c32bc 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/pkgs/__init__.py b/charon/pkgs/__init__.py index dbff095e..9eefcae0 100644 --- a/charon/pkgs/__init__.py +++ b/charon/pkgs/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index bcb5d1f5..d29914f8 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 77b95f52..80760894 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 52477e57..53a226d8 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/storage.py b/charon/storage.py index 99a46d9e..994363cb 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/utils/__init__.py b/charon/utils/__init__.py index dbff095e..9eefcae0 100644 --- a/charon/utils/__init__.py +++ b/charon/utils/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/utils/archive.py b/charon/utils/archive.py index 3dbd2081..ae6f6f3c 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/utils/files.py b/charon/utils/files.py index 22559ab4..d2889b79 100644 --- a/charon/utils/files.py +++ b/charon/utils/files.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/charon/utils/logs.py b/charon/utils/logs.py index 7b0f43c4..582a549d 100644 --- a/charon/utils/logs.py +++ b/charon/utils/logs.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/setup.py b/setup.py index 6d10dbd2..5178e079 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test.sh b/test.sh index bfcff1dc..aa57037d 100755 --- a/test.sh +++ b/test.sh @@ -43,6 +43,9 @@ function setup_charon() { if [[ $OS == "centos" ]]; then # Don't let builddep enable *-source repos since they give 404 errors $RUN rm -f /etc/yum.repos.d/CentOS-Sources.repo + # Resolve "Failed to download metadata for repo: Cannot prepare internal mirrorlist: No URLs in mirrorlist" problem + $RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-AppStream.repo + $RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-BaseOS.repo # This has to run *before* we try installing anything from EPEL $RUN $PKG $ENABLE_REPO install -y epel-release fi diff --git a/tests/base.py b/tests/base.py index 011a5b0f..2249d4e6 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_config.py b/tests/test_config.py index a8aeceec..7014ed5d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 1d03979a..3fb17545 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 79ea0bcf..d5aa221a 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index 74402dbe..af50ea3b 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_maven_meta.py b/tests/test_maven_meta.py index 21d8e409..7c86c6c4 100644 --- a/tests/test_maven_meta.py +++ b/tests/test_maven_meta.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index c1bcb822..fbf34500 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_npm_del.py b/tests/test_npm_del.py index 67a8ec7c..34dcf230 100644 --- a/tests/test_npm_del.py +++ b/tests/test_npm_del.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_npm_index.py b/tests/test_npm_index.py index 9f5e29cb..944126d4 100644 --- a/tests/test_npm_index.py +++ b/tests/test_npm_index.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_npm_meta.py b/tests/test_npm_meta.py index 94de8746..5cff2321 100644 --- a/tests/test_npm_meta.py +++ b/tests/test_npm_meta.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_npm_upload.py b/tests/test_npm_upload.py index 47672fc2..10b8f45b 100644 --- a/tests/test_npm_upload.py +++ b/tests/test_npm_upload.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index c474fda7..1fd5abe5 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_s3client.py b/tests/test_s3client.py index d086c46b..2a7dad80 100644 --- a/tests/test_s3client.py +++ b/tests/test_s3client.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/test_util.py b/tests/test_util.py index be44babf..584920c2 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/charon) +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From caa7e95bfb26b28945d8fcc776aee8e91aaaa4d3 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 16 Feb 2022 14:31:29 +0800 Subject: [PATCH 3/4] Merge pull request #136 from ligangty/main Chore: fix github ci test failure --- .github/workflows/linters.yaml | 2 +- .github/workflows/unittests.yaml | 2 +- test.sh | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index 82deeb2c..94a56e46 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -19,7 +19,7 @@ jobs: matrix: os: - name: centos - version: 8 + version: 7 python: 3 engine: docker diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 1e72ad1f..4c45b551 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -18,7 +18,7 @@ jobs: matrix: os: - name: centos - version: 8 + version: 7 python: 3 engine: docker diff --git a/test.sh b/test.sh index aa57037d..a359cfb7 100755 --- a/test.sh +++ b/test.sh @@ -4,7 +4,7 @@ set -eux # Prepare env vars ENGINE=${ENGINE:="podman"} OS=${OS:="centos"} -OS_VERSION=${OS_VERSION:="8"} +OS_VERSION=${OS_VERSION:="7"} PYTHON_VERSION=${PYTHON_VERSION:="3"} ACTION=${ACTION:="test"} IMAGE="$OS:$OS_VERSION" @@ -33,19 +33,22 @@ function setup_charon() { PKG_EXTRA=(dnf-plugins-core git "$PYTHON"-pylint) BUILDDEP=(dnf builddep) if [[ $OS == "centos" ]]; then + PKG="yum" + PKG_EXTRA=(yum-utils git "$PYTHON"-pylint) + BUILDDEP=(yum-builddep) ENABLE_REPO= else ENABLE_REPO="--enablerepo=updates-testing" fi + PIP_INST=("$PIP" install --index-url "${PYPI_INDEX:-https://pypi.org/simple}") if [[ $OS == "centos" ]]; then # Don't let builddep enable *-source repos since they give 404 errors $RUN rm -f /etc/yum.repos.d/CentOS-Sources.repo - # Resolve "Failed to download metadata for repo: Cannot prepare internal mirrorlist: No URLs in mirrorlist" problem - $RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-AppStream.repo - $RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + # $RUN rm -f /etc/yum.repos.d/CentOS-Linux-AppStream.repo + # $RUN rm -f /etc/yum.repos.d/CentOS-Linux-BaseOS.repo # This has to run *before* we try installing anything from EPEL $RUN $PKG $ENABLE_REPO install -y epel-release fi @@ -69,7 +72,7 @@ function setup_charon() { fi # install with RPM_PY_SYS=true to avoid error caused by installing on system python - $RUN sh -c "RPM_PY_SYS=true ${PIP_INST[*]} rpm-py-installer" + #$RUN sh -c "RPM_PY_SYS=true ${PIP_INST[*]} rpm-py-installer" # Setuptools install charon from source $RUN $PYTHON setup.py install From 05616a2773e5bb19af97ede7ea589762339ff0c7 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 10 Mar 2022 18:15:12 +0800 Subject: [PATCH 4/4] Fix: wrong ignored metadata files of *archetype-catalog.xml --- charon/pkgs/maven.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 80760894..1b9ae27e 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -924,7 +924,7 @@ def __hash_decorate_metadata(path: str, metadata: str) -> List[str]: def _is_ignored(filename: str, ignore_patterns: List[str]) -> bool: for ignored_name in STANDARD_GENERATED_IGNORES: - if ignored_name in filename: + if filename and filename.startswith(ignored_name.strip()): logger.warning("Ignoring standard generated Maven path: %s", filename) return True