From b6d635279d6d6d412452955bb210ed99224aa4b1 Mon Sep 17 00:00:00 2001 From: ROUVREAU Vincent Date: Fri, 8 Jan 2021 16:22:42 +0100 Subject: sphinx shall compile on ws and osx as sphinxcontrib-bibtex is set to version 1.0.0. Add a test strategy file. fix docker warnings at build --- .appveyor.yml | 2 +- .github/for_maintainers/tests_strategy.md | 90 ++++++++++++++++++++++++++++++ .github/test-requirements.txt | 2 +- Dockerfile_for_circleci_image | 2 +- Dockerfile_for_circleci_image_without_cgal | 55 ++++++++++++++++++ Dockerfile_gudhi_installation | 2 +- azure-pipelines.yml | 2 +- 7 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 .github/for_maintainers/tests_strategy.md create mode 100644 Dockerfile_for_circleci_image_without_cgal diff --git a/.appveyor.yml b/.appveyor.yml index 06de5b14..a257debc 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -60,7 +60,7 @@ build_script: cd src/python & type setup.py & MSBuild Cython.sln /m /p:Configuration=Release /p:Platform=x64 & - ctest -j 1 --output-on-failure -C Release -E sphinx + ctest -j 1 --output-on-failure -C Release ) else ( MSBuild GUDHIdev.sln /m /p:Configuration=Release /p:Platform=x64 & ctest -j 1 --output-on-failure -C Release -E diff_files diff --git a/.github/for_maintainers/tests_strategy.md b/.github/for_maintainers/tests_strategy.md new file mode 100644 index 00000000..9c181740 --- /dev/null +++ b/.github/for_maintainers/tests_strategy.md @@ -0,0 +1,90 @@ +# Tests strategy + +This document tries to sum up the tests strategy that has been put in place for gudhi continuous integration. + +The aim is to help maintainers to anticipate third parties modifications, updates. + +## Builds + +### Linux + +As all the third parties are already installed (thanks to docker), the compilations has been seperated by categories to be parallelized: + +* examples (C++) +* tests (C++) +* utils (C++) +* doxygen (C++ documentation that is available in the artefacts) +* python (including documentation and code coverage that are available in the artefacts) + +(cf. `.circleci/config.yml`) + +These build categories are done with and without CGAL, and, with and without Eigen to be sure the users won't be annoyed if a third party is missing. + +With CGAL and with Eigen builds are performed inside the docker image `gudhi/ci_for_gudhi` based on `Dockerfile_for_circleci_image` file. +Without CGAL, and, with or without Eigen builds are performed inside the docker image `gudhi/ci_for_gudhi_wo_cgal` based on `Dockerfile_for_circleci_image_without_cgal` file. + +#### Update docker images + +C++ third parties installation are done thanks to apt on Ubuntu latest LTS. + +Docker images need to be rebuild and push each time `.github/build-requirements`, `.github/test-requirements`, when a new third party is added, when a new CGAL version improves gudhi performances, ... + +```bash +docker build -f Dockerfile_for_circleci_image -t gudhi/ci_for_gudhi:latest . +docker build -f Dockerfile_for_circleci_image_without_cgal -t gudhi/ci_for_gudhi_wo_cgal:latest . +docker login # requires some specific rights on https://hub.docker.com/u/gudhi/repository/docker/gudhi +docker push gudhi/ci_for_gudhi:latest +docker push gudhi/ci_for_gudhi_wo_cgal:latest +``` + +### Windows + +The compilations has been seperated by categories to be parallelized, but I don't know why builds are not run in parallel: + +* examples (C++) +* tests (C++) +* utils (C++) +* python + +Doxygen (C++) is not tested. +(cf. `.appveyor.yml`) + +C++ third parties installation are done thanks to [vcpkg](https://github.com/microsoft/vcpkg/). +In case of installation issue, check in [vcpkg issues](https://github.com/microsoft/vcpkg/issues). + +### OSx + +The compilations has been seperated by categories to be parallelized: + +* examples (C++) +* tests (C++) +* utils (C++) +* python +* Doxygen (C++) + +(cf. `azure-pipelines.yml`) + +C++ third parties installation are done thanks to [brew](https://formulae.brew.sh/formula/). +In case of installation issue, check in formula issues. + +## Pip packaging + +Pip packaging is done in 2 parts: + +* on push and pull requests, the wheels are built (pip package dry-run) +* on releases, the wheels are built and sent to pypi.org (package) + +Only the Linux pip package is based on a docker image (`gudhi/pip_for_gudhi` based on `Dockerfile_for_pip` file) to make it faster. + +### Update docker image + +C++ third parties installation are done thanks to yum on an image based on `quay.io/pypa/manylinux2014_x86_64`. + +Docker image need to be rebuild and push each time `.github/build-requirements`, when a new third party is added, when a new CGAL version improves gudhi performances, ... +As `.github/test-requirements` is not installed, no need to rebuild image when this file is modified. + +```bash +docker build -f Dockerfile_for_pip -t gudhi/pip_for_gudhi:latest . +docker login # requires some specific rights on https://hub.docker.com/u/gudhi/repository/docker/gudhi +docker push gudhi/pip_for_gudhi:latest +``` diff --git a/.github/test-requirements.txt b/.github/test-requirements.txt index 688a2a11..d0803574 100644 --- a/.github/test-requirements.txt +++ b/.github/test-requirements.txt @@ -1,7 +1,7 @@ pytest pytest-cov sphinx -sphinxcontrib-bibtex +sphinxcontrib-bibtex==1.0.0 sphinx-paramlinks matplotlib scipy diff --git a/Dockerfile_for_circleci_image b/Dockerfile_for_circleci_image index f20602b0..60c98f66 100644 --- a/Dockerfile_for_circleci_image +++ b/Dockerfile_for_circleci_image @@ -66,4 +66,4 @@ RUN pip3 install -r build-requirements.txt RUN pip3 --no-cache-dir install -r test-requirements.txt # apt clean up -RUN apt autoremove && rm -rf /var/lib/apt/lists/* +RUN apt-get autoremove && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile_for_circleci_image_without_cgal b/Dockerfile_for_circleci_image_without_cgal new file mode 100644 index 00000000..7bf96667 --- /dev/null +++ b/Dockerfile_for_circleci_image_without_cgal @@ -0,0 +1,55 @@ +FROM ubuntu:20.04 + +# Update and upgrade distribution +RUN apt update && \ + apt upgrade -y + +# Tools necessary for installing and configuring Ubuntu +RUN apt install -y \ + apt-utils \ + locales \ + tzdata + +# Timezone +RUN echo "Europe/Paris" | tee /etc/timezone && \ + ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata + +# Locale with UTF-8 support +RUN echo en_US.UTF-8 UTF-8 >> /etc/locale.gen && \ + locale-gen && \ + update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +# Update again +RUN apt update + +# Required for Gudhi compilation +RUN apt install -y make \ + git \ + g++ \ + cmake \ + perl \ + libboost-all-dev \ + locales \ + python3 \ + python3-pip \ + python3-tk \ + python3-grpcio \ + libfreetype6-dev \ + pkg-config \ + curl + +RUN curl -LO "https://gitlab.com/libeigen/eigen/-/archive/3.3.9/eigen-3.3.9.tar.gz" \ + && tar xf eigen-3.3.9.tar.gz + +ADD .github/build-requirements.txt / +ADD .github/test-requirements.txt / + +RUN pip3 install -r build-requirements.txt +RUN pip3 --no-cache-dir install -r test-requirements.txt + +# apt clean up +RUN apt-get autoremove && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile_gudhi_installation b/Dockerfile_gudhi_installation index ebd21f8d..b0e46d72 100644 --- a/Dockerfile_gudhi_installation +++ b/Dockerfile_gudhi_installation @@ -68,7 +68,7 @@ RUN pip3 install \ scikit-learn # apt clean up -RUN apt autoremove && rm -rf /var/lib/apt/lists/* +RUN apt-get autoremove && rm -rf /var/lib/apt/lists/* RUN curl -LO "https://github.com/GUDHI/gudhi-devel/releases/download/tags%2Fgudhi-release-3.3.0/gudhi.3.3.0.tar.gz" \ && tar xf gudhi.3.3.0.tar.gz \ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 64f3d141..8e88cab5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,5 +33,5 @@ jobs: cmake -DCMAKE_BUILD_TYPE:STRING=$(cmakeBuildType) -DWITH_GUDHI_TEST=ON -DWITH_GUDHI_UTILITIES=ON -DWITH_GUDHI_PYTHON=ON -DPython_ADDITIONAL_VERSIONS=3 .. make -j 4 make doxygen - ctest -j 4 --output-on-failure -E sphinx # remove sphinx build as it fails + ctest -j 4 --output-on-failure # -E sphinx remove sphinx build as it fails displayName: 'Build, test and documentation generation' -- cgit v1.2.3