From 9f6fea1b5aeb3986be6c6881c34115d6810784bf Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Wed, 18 Nov 2020 16:55:30 +0100 Subject: [PATCH 01/28] Add docformatter to pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ab63ab3018..d1cddc8ca82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ ops = ["boto3", "boto3_type_annotations", "paramiko", "docker"] [tool.poetry.dev-dependencies] isort = "==4.3.21" black = "==20.8b1" -docformatter = "==1.3.1" +docformatter = { git = "https://github.com/myint/docformatter", rev = "248f822250486b521b6f737b0b56c6503536b7ea" } mypy = "==0.790" pylint = "==2.5.2" pytest = "==5.3.4" From 32ad1165e95a6eb322865d023e7d960b3816f11e Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Wed, 18 Nov 2020 16:57:39 +0100 Subject: [PATCH 02/28] Add docformatter to test.sh --- dev/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/test.sh b/dev/test.sh index 5238b2b74e2..da6cc739c32 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -6,8 +6,7 @@ echo "=== test.sh ===" isort --skip src/py/flwr/proto --check-only -rc src/py/flwr && echo "- isort: done" && black -q --exclude "src\/py\/flwr\/proto" --check src/py/flwr && echo "- black: done" && -# docformatter is missing --exclude src/py/flwr/proto -# docformatter -c -r src/py/flwr && echo "- docformatter: done" && +docformatter -c -r src/py/flwr -e src/py/flwr/proto && echo "- docformatter: done" && mypy src/py && echo "- mypy: done" && pylint --ignore=src/py/flwr/proto src/py/flwr && echo "- pylint: done" && flake8 src/py/flwr && echo "- flake8: done" && From 3fbaf0505142230da975d45e3cec6140394400d6 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Wed, 18 Nov 2020 16:58:13 +0100 Subject: [PATCH 03/28] Add docformatter to format.sh --- dev/format.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/format.sh b/dev/format.sh index 03f18446af3..5759fd51a5c 100755 --- a/dev/format.sh +++ b/dev/format.sh @@ -5,4 +5,4 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ # Python isort -s src/py/flwr/proto -rc src black -q --exclude src/py/flwr/proto src -docformatter -i -r src/py/flwr +docformatter -i -r src/py/flwr -e src/py/flwr/proto From 55065c8370f446b0b8ee59a26a8724bd43c8d4b4 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Wed, 18 Nov 2020 17:05:17 +0100 Subject: [PATCH 04/28] Add docformatter to CI --- .github/workflows/flower.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flower.yml b/.github/workflows/flower.yml index d9c063d194f..a80eb726f56 100644 --- a/.github/workflows/flower.yml +++ b/.github/workflows/flower.yml @@ -30,7 +30,7 @@ jobs: run: python -m poetry install - name: Check if protos need recompilation run: ./dev/check-protos.sh - - name: Lint + Test (isort/black/mypy/pylint/pytest) + - name: Lint + Test (isort/black/docformatter/mypy/pylint/flake8/pytest) run: ./dev/test.sh test_baseline: From 8362048f7cd428ecead5e8447bec0848e09c99a2 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 10:23:16 +0100 Subject: [PATCH 05/28] add init_py_check.py under src/py/flwr_tool --- src/py/flwr_tool/init_py_check.py | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/py/flwr_tool/init_py_check.py diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py new file mode 100644 index 00000000000..7442be7f151 --- /dev/null +++ b/src/py/flwr_tool/init_py_check.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Copyright 2020 Adap GmbH. All Rights Reserved. + +"""This File will check the specified directory for any missing __init__.py +files.""" + +import os +import re +import sys + + +def check_missing_init_files(absolute_path: str) -> None: + """Searches through the specified absolute_path and looks for missing __init__.py + files.""" + path = os.walk(absolute_path) + warning_list = [] + + for dir_path, _, files_in_dir in path: + # As some directories are automatically + # generated we are going to ignore them + if re.search("__pycache__$", dir_path) is not None: + continue + if re.search(".pytest_cache.*$", dir_path) is not None: + continue + if re.search("dist", dir_path) is not None: + continue + + # If no init is found in current directory add a warning_message to warning_list + if not any(filename == "__init__.py" for filename in files_in_dir): + warning_message = "- " + dir_path + warning_list.append(warning_message) + + if len(warning_list) > 0: + print("Could not find '__init__.py' in the following directories:") + for warning in warning_list: + print(warning) + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + raise Exception( + "Please provide path to directory as in `init_py_check.py src/py/adap`" + ) + abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[1])) + check_missing_init_files(abs_path) From 41b749a9480d928dfb78a9965ca692cfeeaf8dab Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 10:42:05 +0100 Subject: [PATCH 06/28] Update init_py_check.py --- src/py/flwr_tool/init_py_check.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) mode change 100644 => 100755 src/py/flwr_tool/init_py_check.py diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py old mode 100644 new mode 100755 index 7442be7f151..0860317e305 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -14,15 +14,18 @@ def check_missing_init_files(absolute_path: str) -> None: files.""" path = os.walk(absolute_path) warning_list = [] + ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist"] + should_skip: bool = False for dir_path, _, files_in_dir in path: # As some directories are automatically # generated we are going to ignore them - if re.search("__pycache__$", dir_path) is not None: - continue - if re.search(".pytest_cache.*$", dir_path) is not None: - continue - if re.search("dist", dir_path) is not None: + for ignore_word in ignore_list: + if re.search(ignore_word, dir_path) is not None: + should_skip = True + break + if should_skip: + should_skip = False continue # If no init is found in current directory add a warning_message to warning_list From 21d5f268f4cec4202eb048acad089205c4737ed8 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 10:42:49 +0100 Subject: [PATCH 07/28] Add another word to ignore_list --- src/py/flwr_tool/init_py_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 0860317e305..b516e3f9576 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -14,7 +14,7 @@ def check_missing_init_files(absolute_path: str) -> None: files.""" path = os.walk(absolute_path) warning_list = [] - ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist"] + ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$"] should_skip: bool = False for dir_path, _, files_in_dir in path: From 440cc0d0ea06744684ebb6e93f8dadca3ac68c90 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 10:44:10 +0100 Subject: [PATCH 08/28] Add init_py_check.py to test.sh --- dev/test.sh | 1 + src/py/__init__.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/py/__init__.py diff --git a/dev/test.sh b/dev/test.sh index da6cc739c32..b966c55c39e 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -11,4 +11,5 @@ mypy src/py && echo "- mypy: pylint --ignore=src/py/flwr/proto src/py/flwr && echo "- pylint: done" && flake8 src/py/flwr && echo "- flake8: done" && pytest -q src/py/flwr && echo "- pytest: done" && +./src/py/flwr_tool/init_py_check.py src/py && echo "- init-check: done" && echo "- All Python checks passed" diff --git a/src/py/__init__.py b/src/py/__init__.py new file mode 100644 index 00000000000..1304ca888f4 --- /dev/null +++ b/src/py/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 Adap GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flower example using PyTorch for CIFAR-10 image classification.""" + + +DEFAULT_SERVER_ADDRESS = "[::]:8080" From 21eab2fe89f7fe7cf008a1f9125280d17834d610 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 11:13:18 +0100 Subject: [PATCH 09/28] Update init_py_check.py --- src/py/flwr_tool/init_py_check.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index b516e3f9576..25de697e41d 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -15,17 +15,11 @@ def check_missing_init_files(absolute_path: str) -> None: path = os.walk(absolute_path) warning_list = [] ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$"] - should_skip: bool = False for dir_path, _, files_in_dir in path: # As some directories are automatically # generated we are going to ignore them - for ignore_word in ignore_list: - if re.search(ignore_word, dir_path) is not None: - should_skip = True - break - if should_skip: - should_skip = False + if any([re.search(iw, dir_path) is not None for iw in ignore_list]): continue # If no init is found in current directory add a warning_message to warning_list From e5014dffc52ce4e1d6c23cb0a6387246c69173a6 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 13:28:46 +0100 Subject: [PATCH 10/28] Update init_py_check.py --- dev/test.sh | 2 +- src/py/flwr_tool/init_py_check.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dev/test.sh b/dev/test.sh index b966c55c39e..fa02318f01e 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -7,9 +7,9 @@ echo "=== test.sh ===" isort --skip src/py/flwr/proto --check-only -rc src/py/flwr && echo "- isort: done" && black -q --exclude "src\/py\/flwr\/proto" --check src/py/flwr && echo "- black: done" && docformatter -c -r src/py/flwr -e src/py/flwr/proto && echo "- docformatter: done" && +python -m flwr_tool.init_py_check src/py src/py && echo "- init-check: done" && mypy src/py && echo "- mypy: done" && pylint --ignore=src/py/flwr/proto src/py/flwr && echo "- pylint: done" && flake8 src/py/flwr && echo "- flake8: done" && pytest -q src/py/flwr && echo "- pytest: done" && -./src/py/flwr_tool/init_py_check.py src/py && echo "- init-check: done" && echo "- All Python checks passed" diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 25de697e41d..c38705914ed 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -1,8 +1,10 @@ -#!/usr/bin/env python3 # Copyright 2020 Adap GmbH. All Rights Reserved. -"""This File will check the specified directory for any missing __init__.py -files.""" +"""Check provided directory and sub-directories for missing __init__.py files. + +Example: + python -m adap.to.. +""" import os import re @@ -35,7 +37,7 @@ def check_missing_init_files(absolute_path: str) -> None: if __name__ == "__main__": - if len(sys.argv) < 2: + if len(sys.argv) != 2: raise Exception( "Please provide path to directory as in `init_py_check.py src/py/adap`" ) From a3ea3b2a7bf945d1a2983f21c03a45eb42851bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20T=C3=BCrk?= <67055966+CanTuerk@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:29:36 +0100 Subject: [PATCH 11/28] Update test.sh --- dev/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/test.sh b/dev/test.sh index fa02318f01e..c1608b7c245 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -7,7 +7,7 @@ echo "=== test.sh ===" isort --skip src/py/flwr/proto --check-only -rc src/py/flwr && echo "- isort: done" && black -q --exclude "src\/py\/flwr\/proto" --check src/py/flwr && echo "- black: done" && docformatter -c -r src/py/flwr -e src/py/flwr/proto && echo "- docformatter: done" && -python -m flwr_tool.init_py_check src/py src/py && echo "- init-check: done" && +python -m flwr_tool.init_py_check src/py && echo "- init-check: done" && mypy src/py && echo "- mypy: done" && pylint --ignore=src/py/flwr/proto src/py/flwr && echo "- pylint: done" && flake8 src/py/flwr && echo "- flake8: done" && From efd153d3c77f1c3bb4de473d5fb74c645927b09b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20T=C3=BCrk?= <67055966+CanTuerk@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:29:55 +0100 Subject: [PATCH 12/28] Update __init__.py --- src/py/__init__.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/py/__init__.py b/src/py/__init__.py index 1304ca888f4..a96d357d964 100644 --- a/src/py/__init__.py +++ b/src/py/__init__.py @@ -1,18 +1 @@ # Copyright 2020 Adap GmbH. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Flower example using PyTorch for CIFAR-10 image classification.""" - - -DEFAULT_SERVER_ADDRESS = "[::]:8080" From 78664cdf7782f134af2806a40260a757a4d5b03d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20T=C3=BCrk?= <67055966+CanTuerk@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:30:45 +0100 Subject: [PATCH 13/28] Update init_py_check.py --- src/py/flwr_tool/init_py_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index c38705914ed..20749032a88 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -3,7 +3,7 @@ """Check provided directory and sub-directories for missing __init__.py files. Example: - python -m adap.to.. + python -m flwr_tool.init_py_check src/py """ import os @@ -30,7 +30,7 @@ def check_missing_init_files(absolute_path: str) -> None: warning_list.append(warning_message) if len(warning_list) > 0: - print("Could not find '__init__.py' in the following directories:") + print("Could not find '__init__.py' in the following directories:") for warning in warning_list: print(warning) sys.exit(1) From 28c425adbeb223d27d0a3cb28bca2a47384fab29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20T=C3=BCrk?= <67055966+CanTuerk@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:34:46 +0100 Subject: [PATCH 14/28] Delete __init__.py --- src/py/__init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/py/__init__.py diff --git a/src/py/__init__.py b/src/py/__init__.py deleted file mode 100644 index a96d357d964..00000000000 --- a/src/py/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright 2020 Adap GmbH. All Rights Reserved. From 63c3e9f011350edf66b374db31b918055c396d9c Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 13:37:59 +0100 Subject: [PATCH 15/28] add init py --- src/py/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/py/__init__.py diff --git a/src/py/__init__.py b/src/py/__init__.py new file mode 100644 index 00000000000..a87c9e62f22 --- /dev/null +++ b/src/py/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Adap GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== From d5be3769384ca470a819265e56f1c06aa5ce5463 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 13:52:47 +0100 Subject: [PATCH 16/28] update init_py_check.py --- src/py/flwr_tool/init_py_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 20749032a88..88d35087479 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -19,8 +19,7 @@ def check_missing_init_files(absolute_path: str) -> None: ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$"] for dir_path, _, files_in_dir in path: - # As some directories are automatically - # generated we are going to ignore them + # As some directories are automatically generated we are going to ignore them if any([re.search(iw, dir_path) is not None for iw in ignore_list]): continue From e38fc506b7ff4388934c598c1a65b571d55aa1b9 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 14:32:39 +0100 Subject: [PATCH 17/28] Delete useless init file --- src/py/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 src/py/__init__.py diff --git a/src/py/__init__.py b/src/py/__init__.py deleted file mode 100644 index a87c9e62f22..00000000000 --- a/src/py/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2020 Adap GmbH. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== From 2c8654927b9402a1d798395a80bc8246bfbbeab5 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 14:32:53 +0100 Subject: [PATCH 18/28] add another element to ignore_list in init_py_Check.py --- src/py/flwr_tool/init_py_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 88d35087479..76d618a6e17 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -16,7 +16,7 @@ def check_missing_init_files(absolute_path: str) -> None: files.""" path = os.walk(absolute_path) warning_list = [] - ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$"] + ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$", "py$"] for dir_path, _, files_in_dir in path: # As some directories are automatically generated we are going to ignore them From 3f2eba4d0619067d06193a39bdf9240dfe0e3b00 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Tue, 12 Jan 2021 14:42:31 +0100 Subject: [PATCH 19/28] update init_py_check.py --- dev/test.sh | 2 +- src/py/flwr_tool/init_py_check.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dev/test.sh b/dev/test.sh index c1608b7c245..d94e10c6f4a 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -7,7 +7,7 @@ echo "=== test.sh ===" isort --skip src/py/flwr/proto --check-only -rc src/py/flwr && echo "- isort: done" && black -q --exclude "src\/py\/flwr\/proto" --check src/py/flwr && echo "- black: done" && docformatter -c -r src/py/flwr -e src/py/flwr/proto && echo "- docformatter: done" && -python -m flwr_tool.init_py_check src/py && echo "- init-check: done" && +python -m flwr_tool.init_py_check src/py/flwr src/py/flwr_tool src/py/flwr_example src/py/flwr_experimental && echo "- init-check: done" && mypy src/py && echo "- mypy: done" && pylint --ignore=src/py/flwr/proto src/py/flwr && echo "- pylint: done" && flake8 src/py/flwr && echo "- flake8: done" && diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 76d618a6e17..fddf8afeb96 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -36,9 +36,8 @@ def check_missing_init_files(absolute_path: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - raise Exception( - "Please provide path to directory as in `init_py_check.py src/py/adap`" - ) - abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[1])) - check_missing_init_files(abs_path) + for i in range(len(sys.argv)): + if i == 0: + continue + abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) + check_missing_init_files(abs_path) \ No newline at end of file From c29d51fe0b3c2cb08a730304481f9e8cbf407341 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Fri, 15 Jan 2021 09:41:22 +0100 Subject: [PATCH 20/28] Add final newline in init_py_check.py --- src/py/flwr_tool/init_py_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index fddf8afeb96..0ce6a91f4f3 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -40,4 +40,5 @@ def check_missing_init_files(absolute_path: str) -> None: if i == 0: continue abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) - check_missing_init_files(abs_path) \ No newline at end of file + check_missing_init_files(abs_path) + \ No newline at end of file From aa8af43d40cfafc9b462c0d8ed17e0a6e556680c Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Fri, 15 Jan 2021 09:42:35 +0100 Subject: [PATCH 21/28] Remove py from ignore_list --- src/py/flwr_tool/init_py_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 0ce6a91f4f3..b86fc9faef6 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -16,7 +16,7 @@ def check_missing_init_files(absolute_path: str) -> None: files.""" path = os.walk(absolute_path) warning_list = [] - ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$", "py$"] + ignore_list = ["__pycache__$", ".pytest_cache.*$", "dist", "flwr.egg-info$"] for dir_path, _, files_in_dir in path: # As some directories are automatically generated we are going to ignore them @@ -41,4 +41,3 @@ def check_missing_init_files(absolute_path: str) -> None: continue abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) check_missing_init_files(abs_path) - \ No newline at end of file From 93dccaebc47e1a62d1cc5c8c8e136fb2b560ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20T=C3=BCrk?= <67055966+CanTuerk@users.noreply.github.com> Date: Fri, 15 Jan 2021 09:46:07 +0100 Subject: [PATCH 22/28] Update src/py/flwr_tool/init_py_check.py Co-authored-by: Taner Topal --- src/py/flwr_tool/init_py_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 0ce6a91f4f3..2af5ea5e242 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -36,9 +36,9 @@ def check_missing_init_files(absolute_path: str) -> None: if __name__ == "__main__": + if len(sys.argv) == 0: + raise Exception("Please provide at least one directory path relative to your current working directory.") for i in range(len(sys.argv)): - if i == 0: - continue abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) check_missing_init_files(abs_path) - \ No newline at end of file + From 61d9ca487e21080832e29e2072a525051acd5f54 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Fri, 15 Jan 2021 09:48:29 +0100 Subject: [PATCH 23/28] update init_py_check.py --- src/py/flwr_tool/init_py_check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index 62877f39bb5..c154a101d4a 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -37,7 +37,9 @@ def check_missing_init_files(absolute_path: str) -> None: if __name__ == "__main__": if len(sys.argv) == 0: - raise Exception("Please provide at least one directory path relative to your current working directory.") + raise Exception( + "Please provide at least one directory path relative to your current working directory." + ) for i in range(len(sys.argv)): abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) check_missing_init_files(abs_path) From ff41e8e2116e4303f8973c6a541afe52b45e8d96 Mon Sep 17 00:00:00 2001 From: Taner Topal Date: Fri, 15 Jan 2021 12:19:39 +0100 Subject: [PATCH 24/28] Run fnt --- src/py/flwr_example/pytorch_save_weights/client.py | 2 -- src/py/flwr_tool/init_py_check.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/py/flwr_example/pytorch_save_weights/client.py b/src/py/flwr_example/pytorch_save_weights/client.py index 2a4c4caff2a..55084f317c0 100644 --- a/src/py/flwr_example/pytorch_save_weights/client.py +++ b/src/py/flwr_example/pytorch_save_weights/client.py @@ -17,8 +17,6 @@ import argparse import timeit - -# pylint: enable=no-member from typing import Dict, List, Tuple import numpy as np diff --git a/src/py/flwr_tool/init_py_check.py b/src/py/flwr_tool/init_py_check.py index c154a101d4a..8a016a46490 100755 --- a/src/py/flwr_tool/init_py_check.py +++ b/src/py/flwr_tool/init_py_check.py @@ -38,8 +38,8 @@ def check_missing_init_files(absolute_path: str) -> None: if __name__ == "__main__": if len(sys.argv) == 0: raise Exception( - "Please provide at least one directory path relative to your current working directory." - ) + "Please provide at least one directory path relative to your current working directory." + ) for i in range(len(sys.argv)): abs_path: str = os.path.abspath(os.path.join(os.getcwd(), sys.argv[i])) check_missing_init_files(abs_path) From 888c97a92fb8c34d488c5df96ba6e808ce3c9dc0 Mon Sep 17 00:00:00 2001 From: Can Tuerk Date: Mon, 18 Jan 2021 11:36:22 +0100 Subject: [PATCH 25/28] Add missing init file --- src/py/flwr/dataset/utils/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/py/flwr/dataset/utils/__init__.py diff --git a/src/py/flwr/dataset/utils/__init__.py b/src/py/flwr/dataset/utils/__init__.py new file mode 100644 index 00000000000..27276b2292a --- /dev/null +++ b/src/py/flwr/dataset/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Adap GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flower Utils.""" From 99ef5599f63916c6bff6fe518f685d17e9d71ade Mon Sep 17 00:00:00 2001 From: Taner Topal Date: Mon, 18 Jan 2021 16:51:06 +0100 Subject: [PATCH 26/28] Fixes linter errors --- src/py/flwr/dataset/utils/common.py | 3 ++- src/py/flwr/dataset/utils/common_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/dataset/utils/common.py b/src/py/flwr/dataset/utils/common.py index e84c18e8b81..234583cabf8 100644 --- a/src/py/flwr/dataset/utils/common.py +++ b/src/py/flwr/dataset/utils/common.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - +"""Provides commonely used functions when generating a dataset for Federated +Learning.""" # pylint: disable=invalid-name from typing import List, Tuple, cast diff --git a/src/py/flwr/dataset/utils/common_test.py b/src/py/flwr/dataset/utils/common_test.py index b0b96078f37..25717d44a14 100644 --- a/src/py/flwr/dataset/utils/common_test.py +++ b/src/py/flwr/dataset/utils/common_test.py @@ -49,7 +49,7 @@ class CifarPartitionedTestCase(unittest.TestCase): """Tests for partitioned CIFAR-10/100 dataset generation.""" def setUp(self) -> None: - x = np.random.random(size=(500, 3, 32, 32)) + x = np.random.random(size=(500, 3, 32, 32)) # pylint: disable=no-member y = np.concatenate(np.array([50 * [j] for j in range(10)]), axis=0) y = np.expand_dims(y, axis=1) From 2888265588c5086aa8f9662c6da7e5f513e0220c Mon Sep 17 00:00:00 2001 From: "Daniel J. Beutel" Date: Mon, 18 Jan 2021 17:14:36 +0100 Subject: [PATCH 27/28] Update __init__.py --- src/py/flwr/dataset/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/dataset/utils/__init__.py b/src/py/flwr/dataset/utils/__init__.py index 27276b2292a..ee4d0d1a1cb 100644 --- a/src/py/flwr/dataset/utils/__init__.py +++ b/src/py/flwr/dataset/utils/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Flower Utils.""" +"""Flower Datasets utility functions.""" From 6dae1f4c6a215c2c4ea423780e07daeb6d83d1ae Mon Sep 17 00:00:00 2001 From: "Daniel J. Beutel" Date: Mon, 18 Jan 2021 17:16:02 +0100 Subject: [PATCH 28/28] Update common.py --- src/py/flwr/dataset/utils/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/dataset/utils/common.py b/src/py/flwr/dataset/utils/common.py index 234583cabf8..995505a8ed3 100644 --- a/src/py/flwr/dataset/utils/common.py +++ b/src/py/flwr/dataset/utils/common.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Provides commonely used functions when generating a dataset for Federated -Learning.""" +"""Commonly used functions for generating partitioned datasets.""" + # pylint: disable=invalid-name from typing import List, Tuple, cast