From 95bea2182c5f29ecc75437b0f0b742cb816b47df Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Tue, 17 Dec 2019 13:57:29 +0300 Subject: [PATCH 01/19] Remove release date --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bbc55a..3027daa 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ res1.write_async(local_path="~/Downloads/file1", callback) Release Notes ------------- -**Version 0.14 – TBD** +**Version 0.14** * Fixed an issue with checking resources on Yandex WebDAV server **Version 0.13 – 27.11.2019** From 336db8ae19182992e3ad62c4a5f2a70e3be0d57a Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Mon, 20 Jan 2020 17:31:25 -0800 Subject: [PATCH 02/19] Override methods for customizing communication with WebDAV servers. Fix #31 and #30 --- README.md | 23 +++++++++++++++++++++-- tests/base_client_it.py | 5 ++++- tests/test_client_it.py | 3 +++ webdav3/client.py | 5 +---- webdav3/connection.py | 3 ++- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3027daa..53c148d 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ client.execute_request("mkdir", 'directory_name') Webdav API ========== -Webdav API is a set of webdav methods of work with cloud storage. This set includes the following methods: +Webdav API is a set of webdav actions of work with cloud storage. This set includes the following actions: `check`, `free`, `info`, `list`, `mkdir`, `clean`, `copy`, `move`, `download`, `upload`, `publish` and `unpublish`. **Configuring the client** -Required keys for configuring client connection with WevDAV-server are webdav\_hostname and webdav\_login, webdav\_password. +Required keys for configuring client connection with WevDAV-server are `webdav_hostname` and `webdav_login`, `webdav_password`. ```python from webdav3.client import Client @@ -51,6 +51,25 @@ options = { client = Client(options) ``` +If your server does not support `HEAD` method or there are other reasons to override default WebDAV methods for actions use a dictionary option `webdav_override_methods`. +The key should be in the following list: `check`, `free`, `info`, `list`, `mkdir`, `clean`, `copy`, `move`, `download`, `upload`, + `publish` and `unpublish`. The value should a string name of WebDAV method, for example `GET`. + +```python +from webdav3.client import Client + +options = { + 'webdav_hostname': "https://webdav.server.ru", + 'webdav_login': "login", + 'webdav_password': "password", + 'webdav_override_methods': { + 'check': 'GET' + } + +} +client = Client(options) +``` + When a proxy server you need to specify settings to connect through it. ```python diff --git a/tests/base_client_it.py b/tests/base_client_it.py index b8bce2f..3b04cd4 100644 --- a/tests/base_client_it.py +++ b/tests/base_client_it.py @@ -21,7 +21,10 @@ class BaseClientTestCase(unittest.TestCase): options = { 'webdav_hostname': 'http://localhost:8585', 'webdav_login': 'alice', - 'webdav_password': 'secret1234' + 'webdav_password': 'secret1234', + 'webdav_override_methods': { + 'check': 'GET' + } } # options = { diff --git a/tests/test_client_it.py b/tests/test_client_it.py index 900df58..0998fe6 100644 --- a/tests/test_client_it.py +++ b/tests/test_client_it.py @@ -224,6 +224,9 @@ class ClientTestCase(BaseClientTestCase): def test_valid(self): self.assertTrue(self.client.valid()) + def test_check_is_overridden(self): + self.assertEqual('GET', self.client.requests['check']) + if __name__ == '__main__': unittest.main() diff --git a/webdav3/client.py b/webdav3/client.py index 766d0b8..0e85b08 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -153,10 +153,7 @@ class Client(object): webdav_options = get_options(option_type=WebDAVSettings, from_options=options) self.webdav = WebDAVSettings(webdav_options) - response = self.execute_request('options', '') - self.supported_methods = response.headers.get('Allow') - if 'HEAD' not in self.supported_methods: - self.requests['check'] = 'GET' + self.requests.update(self.webdav.override_methods) self.default_options = {} def get_headers(self, action, headers_ext=None): diff --git a/webdav3/connection.py b/webdav3/connection.py index ce99211..a122f86 100644 --- a/webdav3/connection.py +++ b/webdav3/connection.py @@ -25,7 +25,7 @@ class WebDAVSettings(ConnectionSettings): ns = "webdav:" prefix = "webdav_" keys = {'hostname', 'login', 'password', 'token', 'root', 'cert_path', 'key_path', 'recv_speed', 'send_speed', - 'verbose', 'disable_check'} + 'verbose', 'disable_check', 'override_methods'} hostname = None login = None @@ -38,6 +38,7 @@ class WebDAVSettings(ConnectionSettings): send_speed = None verbose = None disable_check = False + override_methods = {} def __init__(self, options): self.options = dict() From fc14ed2be1252e5c278d14e065552d85d25aec3a Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Mon, 20 Jan 2020 17:58:31 -0800 Subject: [PATCH 03/19] Update dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c7dde81..5d14233 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( version=version, packages=find_packages(), requires=['python (>= 2.7.6)'], - install_requires=['requests', 'lxml', 'argcomplete'], + install_requires=['requests', 'lxml'], scripts=['wdc'], test_suite='tests', tests_require=['pytest'], From f3e7d4427669ed8dd8f940d95e8f83a5638406f1 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 26 Jan 2020 15:34:05 -0800 Subject: [PATCH 04/19] Support multiple clients simultaneously Fixed #34 --- tests/test_multi_client_it.py | 26 ++++++++++++++++++++++++++ webdav3/client.py | 18 +++++++++--------- webdav3/connection.py | 26 +++++++++++++------------- 3 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 tests/test_multi_client_it.py diff --git a/tests/test_multi_client_it.py b/tests/test_multi_client_it.py new file mode 100644 index 0000000..d8a1d15 --- /dev/null +++ b/tests/test_multi_client_it.py @@ -0,0 +1,26 @@ +import unittest + +from tests.test_client_it import ClientTestCase +from webdav3.client import Client + + +class MultiClientTestCase(ClientTestCase): + options2 = { + 'webdav_hostname': 'https://wrong.url.ru', + 'webdav_login': 'webdavclient.test2', + 'webdav_password': 'Qwerty123!', + 'webdav_override_methods': { + 'check': 'FAKE', + 'download': 'FAKE', + 'upload': 'FAKE', + 'clean': 'FAKE', + } + } + + def setUp(self): + super(ClientTestCase, self).setUp() + self.second_client = Client(self.options2) + + +if __name__ == '__main__': + unittest.main() diff --git a/webdav3/client.py b/webdav3/client.py index 0e85b08..8bfe688 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -89,11 +89,8 @@ class Client(object): # controls whether to verify the server's TLS certificate or not verify = True - # Sets the session for subsequent requests - session = requests.Session() - # HTTP headers for different actions - http_header = { + default_http_header = { 'list': ["Accept: */*", "Depth: 1"], 'free': ["Accept: */*", "Depth: 0", "Content-Type: text/xml"], 'copy': ["Accept: */*"], @@ -107,7 +104,7 @@ class Client(object): } # mapping of actions to WebDAV methods - requests = { + default_requests = { 'options': 'OPTIONS', 'download': "GET", 'upload': "PUT", @@ -150,6 +147,9 @@ class Client(object): `webdav_verbose`: (optional) set verbose mode on.off. By default verbose mode is off. """ + self.session = requests.Session() + self.http_header = Client.default_http_header.copy() + self.requests = Client.default_requests.copy() webdav_options = get_options(option_type=WebDAVSettings, from_options=options) self.webdav = WebDAVSettings(webdav_options) @@ -164,11 +164,11 @@ class Client(object): the specified action. :return: the dictionary of headers for specified action. """ - if action in Client.http_header: + if action in self.http_header: try: - headers = Client.http_header[action].copy() + headers = self.http_header[action].copy() except AttributeError: - headers = Client.http_header[action][:] + headers = self.http_header[action][:] else: headers = list() @@ -211,7 +211,7 @@ class Client(object): if self.session.auth: self.session.request(method="GET", url=self.webdav.hostname, verify=self.verify) # (Re)Authenticates against the proxy response = self.session.request( - method=Client.requests[action], + method=self.requests[action], url=self.get_url(path), auth=(self.webdav.login, self.webdav.password), headers=self.get_headers(action, headers_ext), diff --git a/webdav3/connection.py b/webdav3/connection.py index a122f86..302212b 100644 --- a/webdav3/connection.py +++ b/webdav3/connection.py @@ -27,20 +27,20 @@ class WebDAVSettings(ConnectionSettings): keys = {'hostname', 'login', 'password', 'token', 'root', 'cert_path', 'key_path', 'recv_speed', 'send_speed', 'verbose', 'disable_check', 'override_methods'} - hostname = None - login = None - password = None - token = None - root = None - cert_path = None - key_path = None - recv_speed = None - send_speed = None - verbose = None - disable_check = False - override_methods = {} - def __init__(self, options): + self.hostname = None + self.login = None + self.password = None + self.token = None + self.root = None + self.cert_path = None + self.key_path = None + self.recv_speed = None + self.send_speed = None + self.verbose = None + self.disable_check = False + self.override_methods = {} + self.options = dict() for key in self.keys: From c380dc56c71a14c94132357b0d9702bcb23b5ac0 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 26 Jan 2020 19:05:33 -0800 Subject: [PATCH 05/19] Update readme release notes --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 53c148d..71a963f 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,10 @@ res1.write_async(local_path="~/Downloads/file1", callback) Release Notes ------------- +**Version 3.14** + * Override methods for customizing communication with WebDAV servers + * Support multiple clients simultaneously + **Version 0.14** * Fixed an issue with checking resources on Yandex WebDAV server From 3d4751cdd03f682aa539b764f45beabaf5ffde33 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 26 Jan 2020 19:17:03 -0800 Subject: [PATCH 06/19] Support Python 3.8 --- .travis.yml | 6 +++++- setup.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0277e80..3e5e1e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ services: python: - "2.7" - "3.7" + - "3.8" before_install: - docker pull bytemark/webdav @@ -25,4 +26,7 @@ install: script: - coverage run setup.py test - coverage xml - - sonar-scanner + - | + if [[ $TRAVIS_PYTHON_VERSION == "3.8" ]]; then + sonar-scanner + fi diff --git a/setup.py b/setup.py index 5d14233..dc99d68 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ setup( 'Operating System :: Unix', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Internet', 'Topic :: Software Development :: Libraries :: Python Modules', ], From b3c753c1269b7f1261e5dfa95815550a0ada5223 Mon Sep 17 00:00:00 2001 From: Ishak BELAHMAR Date: Sat, 11 Jan 2020 08:08:38 -0800 Subject: [PATCH 07/19] Handle 2-way sync when files are modified --- setup.py | 2 +- webdav3/client.py | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index dc99d68..8f7e190 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( version=version, packages=find_packages(), requires=['python (>= 2.7.6)'], - install_requires=['requests', 'lxml'], + install_requires=['requests', 'lxml', 'python-dateutil'], scripts=['wdc'], test_suite='tests', tests_require=['pytest'], diff --git a/webdav3/client.py b/webdav3/client.py index 8bfe688..420f1af 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -10,6 +10,7 @@ from re import sub import lxml.etree as etree import requests +from dateutil import parser as dateutil_parser from webdav3.connection import * from webdav3.exceptions import * @@ -697,11 +698,12 @@ class Client(object): self.push(remote_directory=remote_path, local_directory=local_path) else: if local_resource_name in remote_resource_names: - continue + if self.is_local_more_recent(local_path, remote_path): + continue self.upload_file(remote_path=remote_path, local_path=local_path) - def pull(self, remote_directory, local_directory): + def pull(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] @@ -736,13 +738,39 @@ class Client(object): self.pull(remote_directory=remote_path, local_directory=local_path) else: if remote_resource_name in local_resource_names: - continue + # Skip pull if local resource is more recent of we can't tell + if self.is_local_more_recent(local_path, remote_path) in (True, None): + continue + self.download_file(remote_path=remote_path, local_path=local_path) updated = True return updated - def sync(self, remote_directory, local_directory): + def is_local_more_recent(self, local_path, remote_path): + """Tells if local resource is more recent that the remote on if possible + :param local_path: the path to local resource. + :param remote_path: the path to remote resource. + + :return: True if local resource is more recent, False if the remote one is + None if comparison is not possible + """ + try: + remote_info = self.info(remote_path) + # Try to compare modification dates if our server + # offers that information + remote_last_mod_date = remote_info['modified'] + remote_last_mod_date = dateutil_parser.parse(remote_last_mod_date) + remote_last_mod_date_unix_ts = int(remote_last_mod_date.strftime("%s")) + local_last_mod_date_unix_ts = os.stat(local_path).st_mtime + + return (local_last_mod_date_unix_ts > remote_last_mod_date_unix_ts) + except: + # If there is problem when parsing dates, or cannot get + # last modified information, return None + return None + + def sync(self, remote_directory, local_directory): self.pull(remote_directory=remote_directory, local_directory=local_directory) self.push(remote_directory=remote_directory, local_directory=local_directory) From 85b10d53c926c7e4a0ae053b821bf9345643515a Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 26 Jan 2020 20:03:58 -0800 Subject: [PATCH 08/19] Update release notes --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 71a963f..8db0bda 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,7 @@ Release Notes **Version 3.14** * Override methods for customizing communication with WebDAV servers * Support multiple clients simultaneously + * Sync modified files during pull and push by https://github.com/mont5piques **Version 0.14** * Fixed an issue with checking resources on Yandex WebDAV server From 8c7ee2e372a6a4be787f4278d8561722e89d9e2c Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 26 Jan 2020 21:24:26 -0800 Subject: [PATCH 09/19] Fix issues and tests --- tests/base_client_it.py | 44 +++++++++++++++---------- tests/test_client_it.py | 51 +++++++++++++++++++++++++++++ webdav3/client.py | 72 +++++++++++++++++++---------------------- 3 files changed, 111 insertions(+), 56 deletions(-) diff --git a/tests/base_client_it.py b/tests/base_client_it.py index 3b04cd4..9fa988c 100644 --- a/tests/base_client_it.py +++ b/tests/base_client_it.py @@ -35,29 +35,39 @@ class BaseClientTestCase(unittest.TestCase): def setUp(self): self.client = Client(self.options) - if path.exists(path=self.local_path_dir): - shutil.rmtree(path=self.local_path_dir) + self.clean_local_dir(self.local_path_dir) def tearDown(self): - if path.exists(path=self.local_path_dir): - shutil.rmtree(path=self.local_path_dir) - if self.client.check(remote_path=self.remote_path_dir): - self.client.clean(remote_path=self.remote_path_dir) - if self.client.check(remote_path=self.remote_path_dir2): - self.client.clean(remote_path=self.remote_path_dir2) + self.clean_local_dir(self.local_path_dir) + self.clean_remote_dir(self.remote_path_dir) + self.clean_remote_dir(self.remote_path_dir2) - def _prepare_for_downloading(self, inner_dir=False): - if not self.client.check(remote_path=self.remote_path_dir): - self.client.mkdir(remote_path=self.remote_path_dir) - if not self.client.check(remote_path=self.remote_path_file): - self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) + def clean_remote_dir(self, remote_path_dir): + if self.client.check(remote_path=remote_path_dir): + self.client.clean(remote_path=remote_path_dir) + + @staticmethod + def clean_local_dir(local_path_dir): + if path.exists(path=local_path_dir): + shutil.rmtree(path=local_path_dir) + + def _prepare_for_downloading(self, inner_dir=False, base_path=''): + if base_path: + self._create_remote_dir_if_needed(base_path) + self._prepare_dir_for_downloading(base_path + self.remote_path_dir, base_path + self.remote_path_file, self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) if inner_dir: - if not self.client.check(remote_path=self.remote_inner_path_dir): - self.client.mkdir(remote_path=self.remote_inner_path_dir) - if not self.client.check(remote_path=self.remote_inner_path_file): - self.client.upload_file(remote_path=self.remote_inner_path_file, local_path=self.local_file_path) + self._prepare_dir_for_downloading(base_path + self.remote_inner_path_dir, base_path + self.remote_inner_path_file, self.local_file_path) + + def _prepare_dir_for_downloading(self, remote_path_dir, remote_path_file, local_file_path): + self._create_remote_dir_if_needed(remote_path_dir) + if not self.client.check(remote_path=remote_path_file): + self.client.upload_file(remote_path=remote_path_file, local_path=local_file_path) + + def _create_remote_dir_if_needed(self, remote_dir): + if not self.client.check(remote_path=remote_dir): + self.client.mkdir(remote_path=remote_dir) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): diff --git a/tests/test_client_it.py b/tests/test_client_it.py index 0998fe6..2d6c14f 100644 --- a/tests/test_client_it.py +++ b/tests/test_client_it.py @@ -1,4 +1,5 @@ import os.path +import shutil import unittest from io import BytesIO, StringIO from os import path @@ -9,6 +10,7 @@ from webdav3.exceptions import MethodNotSupported, OptionNotValid, RemoteResourc class ClientTestCase(BaseClientTestCase): + pulled_file = BaseClientTestCase.local_path_dir + os.sep + BaseClientTestCase.local_file def test_list(self): self._prepare_for_downloading() @@ -227,6 +229,55 @@ class ClientTestCase(BaseClientTestCase): def test_check_is_overridden(self): self.assertEqual('GET', self.client.requests['check']) + def test_pull_newer(self): + init_modification_time = int(self._prepare_local_test_file_and_get_modification_time()) + sleep(1) + self._prepare_for_downloading(base_path='time/') + result = self.client.pull('time/' + self.remote_path_dir, self.local_path_dir) + update_modification_time = int(os.path.getmtime(self.pulled_file)) + self.assertTrue(result) + self.assertGreater(update_modification_time, init_modification_time) + self.client.clean(remote_path='time/' + self.remote_path_dir) + + def test_pull_older(self): + self._prepare_for_downloading(base_path='time/') + sleep(1) + init_modification_time = int(self._prepare_local_test_file_and_get_modification_time()) + result = self.client.pull('time/' + self.remote_path_dir, self.local_path_dir) + update_modification_time = int(os.path.getmtime(self.pulled_file)) + self.assertFalse(result) + self.assertEqual(update_modification_time, init_modification_time) + self.client.clean(remote_path='time/' + self.remote_path_dir) + + def test_push_newer(self): + self._prepare_for_downloading(base_path='time/') + sleep(1) + self._prepare_for_uploading() + init_modification_time = self.client.info('time/' + self.remote_path_file)['modified'] + result = self.client.push('time/' + self.remote_path_dir, self.local_path_dir) + update_modification_time = self.client.info('time/' + self.remote_path_file)['modified'] + self.assertTrue(result) + self.assertNotEqual(init_modification_time, update_modification_time) + self.client.clean(remote_path='time/' + self.remote_path_dir) + + def test_push_older(self): + self._prepare_for_uploading() + sleep(1) + self._prepare_for_downloading(base_path='time/') + init_modification_time = self.client.info('time/' + self.remote_path_file)['modified'] + result = self.client.push('time/' + self.remote_path_dir, self.local_path_dir) + update_modification_time = self.client.info('time/' + self.remote_path_file)['modified'] + self.assertFalse(result) + self.assertEqual(init_modification_time, update_modification_time) + self.client.clean(remote_path='time/' + self.remote_path_dir) + + def _prepare_local_test_file_and_get_modification_time(self): + if not path.exists(path=self.local_path_dir): + os.mkdir(self.local_path_dir) + if not path.exists(path=self.local_path_dir + os.sep + self.local_file): + shutil.copy(src=self.local_file_path, dst=self.pulled_file) + return os.path.getmtime(self.pulled_file) + if __name__ == '__main__': unittest.main() diff --git a/webdav3/client.py b/webdav3/client.py index 420f1af..a5da364 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -671,51 +671,39 @@ class Client(object): def prune(src, exp): return [sub(exp, "", item) for item in src] + updated = False urn = Urn(remote_directory, directory=True) - - if not self.is_dir(urn.path()): - raise OptionNotValid(name="remote_path", value=remote_directory) - - if not os.path.isdir(local_directory): - raise OptionNotValid(name="local_path", value=local_directory) - - if not os.path.exists(local_directory): - raise LocalResourceNotFound(local_directory) + self._validate_remote_directory(urn) + self._validate_local_directory(local_directory) paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=urn.path()) remote_resource_names = prune(paths, expression) for local_resource_name in listdir(local_directory): - local_path = os.path.join(local_directory, local_resource_name) - remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), - resource_name=local_resource_name) + remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), resource_name=local_resource_name) if os.path.isdir(local_path): if not self.check(remote_path=remote_path): self.mkdir(remote_path=remote_path) - self.push(remote_directory=remote_path, local_directory=local_path) + result = self.push(remote_directory=remote_path, local_directory=local_path) + updated = updated or result else: - if local_resource_name in remote_resource_names: - if self.is_local_more_recent(local_path, remote_path): - continue + if local_resource_name in remote_resource_names and not self.is_local_more_recent(local_path, remote_path): + continue self.upload_file(remote_path=remote_path, local_path=local_path) - + updated = True + return updated def pull(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] updated = False - urn = Urn(remote_directory, directory=True) - - if not self.is_dir(urn.path()): - raise OptionNotValid(name="remote_path", value=remote_directory) - - if not os.path.exists(local_directory): - raise LocalResourceNotFound(local_directory) + self._validate_remote_directory(urn) + self._validate_local_directory(local_directory) local_resource_names = listdir(local_directory) @@ -724,23 +712,19 @@ class Client(object): remote_resource_names = prune(paths, expression) for remote_resource_name in remote_resource_names: - local_path = os.path.join(local_directory, remote_resource_name) - remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), - resource_name=remote_resource_name) - + remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), resource_name=remote_resource_name) remote_urn = Urn(remote_path) if remote_urn.path().endswith("/"): if not os.path.exists(local_path): updated = True os.mkdir(local_path) - self.pull(remote_directory=remote_path, local_directory=local_path) + result = self.pull(remote_directory=remote_path, local_directory=local_path) + updated = updated or result else: - if remote_resource_name in local_resource_names: - # Skip pull if local resource is more recent of we can't tell - if self.is_local_more_recent(local_path, remote_path) in (True, None): - continue + if remote_resource_name in local_resource_names and self.is_local_more_recent(local_path, remote_path): + continue self.download_file(remote_path=remote_path, local_path=local_path) updated = True @@ -757,15 +741,13 @@ class Client(object): """ try: remote_info = self.info(remote_path) - # Try to compare modification dates if our server - # offers that information remote_last_mod_date = remote_info['modified'] remote_last_mod_date = dateutil_parser.parse(remote_last_mod_date) - remote_last_mod_date_unix_ts = int(remote_last_mod_date.strftime("%s")) - local_last_mod_date_unix_ts = os.stat(local_path).st_mtime + remote_last_mod_date_unix_ts = int(remote_last_mod_date.timestamp()) + local_last_mod_date_unix_ts = int(os.stat(local_path).st_mtime) - return (local_last_mod_date_unix_ts > remote_last_mod_date_unix_ts) - except: + return remote_last_mod_date_unix_ts < local_last_mod_date_unix_ts + except ValueError or RuntimeWarning or KeyError: # If there is problem when parsing dates, or cannot get # last modified information, return None return None @@ -774,6 +756,18 @@ class Client(object): self.pull(remote_directory=remote_directory, local_directory=local_directory) self.push(remote_directory=remote_directory, local_directory=local_directory) + def _validate_remote_directory(self, urn): + if not self.is_dir(urn.path()): + raise OptionNotValid(name="remote_path", value=urn.path()) + + @staticmethod + def _validate_local_directory(local_directory): + if not os.path.isdir(local_directory): + raise OptionNotValid(name="local_path", value=local_directory) + + if not os.path.exists(local_directory): + raise LocalResourceNotFound(local_directory) + class Resource(object): def __init__(self, client, urn): From fe4e56caad796b2fe3f20fb9017194d4ca1c79e2 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Wed, 29 Jan 2020 22:34:16 -0800 Subject: [PATCH 10/19] Cancel support Python earlie 3.5 --- .travis.yml | 3 ++- setup.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e5e1e4..1bbd775 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ services: - docker python: - - "2.7" + - "3.5" + - "3.6" - "3.7" - "3.8" diff --git a/setup.py b/setup.py index 8f7e190..e4c41e7 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ setup( name='webdavclient3', version=version, packages=find_packages(), - requires=['python (>= 2.7.6)'], + requires=['python (>= 3.3.0)'], install_requires=['requests', 'lxml', 'python-dateutil'], scripts=['wdc'], test_suite='tests', @@ -73,7 +73,8 @@ setup( 'Operating System :: MacOS', 'Operating System :: Microsoft', 'Operating System :: Unix', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Topic :: Internet', From eced3b5fe5d9777b9f7899d6732c643008caab5e Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Fri, 31 Jan 2020 21:02:43 -0800 Subject: [PATCH 11/19] Bump version to 3.14 and prepare to release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4c41e7..fcbf7b2 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup, find_packages from setuptools.command.install import install as InstallCommand from setuptools.command.test import test as TestCommand -version = "0.14" +version = "3.14" requirements = "libxml2-dev libxslt-dev python-dev" From f64efedd96ffe3c844f1fa3b3093b32c52bfe932 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Tue, 18 Feb 2020 16:47:02 -0800 Subject: [PATCH 12/19] Fixed issue #40 - error during coping and moving files with cyrillic names --- README.md | 3 +++ tests/base_client_it.py | 1 + tests/test_client_it.py | 6 +++--- tests/test_cyrilic_client_it.py | 23 +++++++++++++++++++++++ tests/тестовый.txt | 1 + webdav3/client.py | 4 ++-- 6 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 tests/test_cyrilic_client_it.py create mode 100644 tests/тестовый.txt diff --git a/README.md b/README.md index 8db0bda..0445a5a 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,9 @@ res1.write_async(local_path="~/Downloads/file1", callback) Release Notes ------------- +**Version 3.14.1** + * Fixed issue during coping and moving files with cyrillic names + **Version 3.14** * Override methods for customizing communication with WebDAV servers * Support multiple clients simultaneously diff --git a/tests/base_client_it.py b/tests/base_client_it.py index 9fa988c..5b4290f 100644 --- a/tests/base_client_it.py +++ b/tests/base_client_it.py @@ -13,6 +13,7 @@ class BaseClientTestCase(unittest.TestCase): remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' remote_inner_path_dir = 'test_dir/inner' + inner_dir_name = 'inner' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' diff --git a/tests/test_client_it.py b/tests/test_client_it.py index 2d6c14f..515cca6 100644 --- a/tests/test_client_it.py +++ b/tests/test_client_it.py @@ -208,10 +208,10 @@ class ClientTestCase(BaseClientTestCase): self._prepare_for_downloading(True) self.client.pull(self.remote_path_dir, self.local_path_dir) self.assertTrue(path.exists(self.local_path_dir), 'Expected the directory is downloaded.') - self.assertTrue(path.exists(self.local_path_dir + os.path.sep + 'inner'), 'Expected the directory is downloaded.') - self.assertTrue(path.exists(self.local_path_dir + os.path.sep + 'inner'), 'Expected the directory is downloaded.') + self.assertTrue(path.exists(self.local_path_dir + os.path.sep + self.inner_dir_name), 'Expected the directory is downloaded.') + self.assertTrue(path.exists(self.local_path_dir + os.path.sep + self.inner_dir_name), 'Expected the directory is downloaded.') self.assertTrue(path.isdir(self.local_path_dir), 'Expected this is a directory.') - self.assertTrue(path.isdir(self.local_path_dir + os.path.sep + 'inner'), 'Expected this is a directory.') + self.assertTrue(path.isdir(self.local_path_dir + os.path.sep + self.inner_dir_name), 'Expected this is a directory.') self.assertTrue(path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue(path.isfile(self.local_path_dir + os.path.sep + self.local_file), diff --git a/tests/test_cyrilic_client_it.py b/tests/test_cyrilic_client_it.py new file mode 100644 index 0000000..87add07 --- /dev/null +++ b/tests/test_cyrilic_client_it.py @@ -0,0 +1,23 @@ +import os +import unittest + +from tests.test_client_it import ClientTestCase + + +class MultiClientTestCase(ClientTestCase): + remote_path_file = 'директория/тестовый.txt' + remote_path_file2 = 'директория/тестовый2.txt' + remote_inner_path_file = 'директория/вложенная/тестовый.txt' + remote_path_dir = 'директория' + remote_path_dir2 = 'директория2' + remote_inner_path_dir = 'директория/вложенная' + inner_dir_name = 'вложенная' + local_base_dir = 'tests/' + local_file = 'тестовый.txt' + local_file_path = local_base_dir + 'тестовый.txt' + local_path_dir = local_base_dir + 'res/директория' + pulled_file = local_path_dir + os.sep + local_file + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/тестовый.txt b/tests/тестовый.txt new file mode 100644 index 0000000..1cc4810 --- /dev/null +++ b/tests/тестовый.txt @@ -0,0 +1 @@ +test content for testing of webdav client \ No newline at end of file diff --git a/webdav3/client.py b/webdav3/client.py index a5da364..0768b41 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -535,7 +535,7 @@ class Client(object): raise RemoteParentNotFound(urn_to.path()) headers = [ - "Destination: {url}".format(url=self.get_url(urn_to)) + "Destination: {url}".format(url=self.get_url(urn_to.quote())) ] if self.is_dir(urn_from.path()): headers.append("Depth: {depth}".format(depth=depth)) @@ -558,7 +558,7 @@ class Client(object): if not self.check(urn_to.parent()): raise RemoteParentNotFound(urn_to.path()) - header_destination = "Destination: {path}".format(path=self.get_url(urn_to)) + header_destination = "Destination: {path}".format(path=self.get_url(urn_to.quote())) header_overwrite = "Overwrite: {flag}".format(flag="T" if overwrite else "F") self.execute_request(action='move', path=urn_from.quote(), headers_ext=[header_destination, header_overwrite]) From 3c5ba516af8ae198f00632a72dbd28f8b0343656 Mon Sep 17 00:00:00 2001 From: Daniel Loader Date: Fri, 14 Feb 2020 09:34:38 +0000 Subject: [PATCH 13/19] Add Oauth2 Bearer Token support Basic implementation of authenticating with token instead of basic auth, as the auth=(login,password) flag on requests sends junk authentication as it overrides the bearer authentication header with a base64 encoded blank string. --- webdav3/client.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/webdav3/client.py b/webdav3/client.py index 0768b41..94e212c 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -177,7 +177,7 @@ class Client(object): headers.extend(headers_ext) if self.webdav.token: - webdav_token = "Authorization: OAuth {token}".format(token=self.webdav.token) + webdav_token = "Authorization: Bearer {token}".format(token=self.webdav.token) headers.append(webdav_token) return dict([map(lambda s: s.strip(), i.split(':', 1)) for i in headers]) @@ -211,16 +211,27 @@ class Client(object): """ if self.session.auth: self.session.request(method="GET", url=self.webdav.hostname, verify=self.verify) # (Re)Authenticates against the proxy - response = self.session.request( - method=self.requests[action], - url=self.get_url(path), - auth=(self.webdav.login, self.webdav.password), - headers=self.get_headers(action, headers_ext), - timeout=self.timeout, - data=data, - stream=True, - verify=self.verify - ) + if all([self.webdav.login, self.webdav.password]) is False: + response = self.session.request( + method=self.requests[action], + url=self.get_url(path), + headers=self.get_headers(action, headers_ext), + timeout=self.timeout, + data=data, + stream=True, + verify=self.verify + ) + else: + response = self.session.request( + method=self.requests[action], + url=self.get_url(path), + auth=(self.webdav.login, self.webdav.password), + headers=self.get_headers(action, headers_ext), + timeout=self.timeout, + data=data, + stream=True, + verify=self.verify + ) if response.status_code == 507: raise NotEnoughSpace() if response.status_code == 404: From 0ac3c5e55faafb986c4f0c7ec4c1da39ba545780 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Tue, 18 Feb 2020 17:19:57 -0800 Subject: [PATCH 14/19] Small refactoring --- README.md | 1 + webdav3/client.py | 31 ++++++++++--------------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0445a5a..a6eef1a 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,7 @@ Release Notes ------------- **Version 3.14.1** * Fixed issue during coping and moving files with cyrillic names + * Support OAuth2 bearer tokens by https://github.com/danielloader **Version 3.14** * Override methods for customizing communication with WebDAV servers diff --git a/webdav3/client.py b/webdav3/client.py index 94e212c..ca29f80 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -211,27 +211,16 @@ class Client(object): """ if self.session.auth: self.session.request(method="GET", url=self.webdav.hostname, verify=self.verify) # (Re)Authenticates against the proxy - if all([self.webdav.login, self.webdav.password]) is False: - response = self.session.request( - method=self.requests[action], - url=self.get_url(path), - headers=self.get_headers(action, headers_ext), - timeout=self.timeout, - data=data, - stream=True, - verify=self.verify - ) - else: - response = self.session.request( - method=self.requests[action], - url=self.get_url(path), - auth=(self.webdav.login, self.webdav.password), - headers=self.get_headers(action, headers_ext), - timeout=self.timeout, - data=data, - stream=True, - verify=self.verify - ) + response = self.session.request( + method=self.requests[action], + url=self.get_url(path), + auth=(self.webdav.login, self.webdav.password) if not self.webdav.token else None, + headers=self.get_headers(action, headers_ext), + timeout=self.timeout, + data=data, + stream=True, + verify=self.verify + ) if response.status_code == 507: raise NotEnoughSpace() if response.status_code == 404: From 3aab20b02a00213bb6265f467a7e0527d9979f68 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 23 Feb 2020 17:43:12 -0800 Subject: [PATCH 15/19] Bump version to 3.14.1 before publish --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fcbf7b2..abd81c1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup, find_packages from setuptools.command.install import install as InstallCommand from setuptools.command.test import test as TestCommand -version = "3.14" +version = "3.14.1" requirements = "libxml2-dev libxslt-dev python-dev" From af64110364af2253e3afd5e7f9f62982fa5a8be1 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 23 Mar 2020 10:34:34 +0100 Subject: [PATCH 16/19] Fix request calling to use session's auth. --- webdav3/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webdav3/client.py b/webdav3/client.py index ca29f80..c88a172 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -214,7 +214,7 @@ class Client(object): response = self.session.request( method=self.requests[action], url=self.get_url(path), - auth=(self.webdav.login, self.webdav.password) if not self.webdav.token else None, + auth=(self.webdav.login, self.webdav.password) if (not self.webdav.token and not self.session.auth) else None, headers=self.get_headers(action, headers_ext), timeout=self.timeout, data=data, From eeaf66a2788ca42ec93ed5b0410c391e28badfde Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 5 Apr 2020 17:10:49 -0700 Subject: [PATCH 17/19] Update Travis CI config for SonarQube --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bbd775..0873913 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,7 @@ dist: xenial addons: sonarcloud: organization: "ezhov-evgeny" - token: - secure: f0f714f3bea6bd103e3eb82724ef3bb0d3b54d1d + token: f0f714f3bea6bd103e3eb82724ef3bb0d3b54d1d services: - docker From 39afefffc5e7b5613786dcaa2d719c50d847e703 Mon Sep 17 00:00:00 2001 From: Evgeny Ezhov Date: Sun, 5 Apr 2020 17:43:10 -0700 Subject: [PATCH 18/19] Fixed #49: Remove tailing slashes in web_hostname --- tests/test_tailing_slash_client_it.py | 26 ++++++++++++++++++++++++++ webdav3/connection.py | 1 + webdav3/urn.py | 2 -- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tests/test_tailing_slash_client_it.py diff --git a/tests/test_tailing_slash_client_it.py b/tests/test_tailing_slash_client_it.py new file mode 100644 index 0000000..6a8744c --- /dev/null +++ b/tests/test_tailing_slash_client_it.py @@ -0,0 +1,26 @@ +import unittest + +from tests.test_client_it import ClientTestCase + + +class TailingSlashClientTestCase(ClientTestCase): + options = { + 'webdav_hostname': 'http://localhost:8585/', + 'webdav_login': 'alice', + 'webdav_password': 'secret1234', + 'webdav_override_methods': { + 'check': 'GET' + } + } + + def test_list_inner(self): + self._prepare_for_downloading(True) + file_list = self.client.list(self.remote_inner_path_dir) + self.assertIsNotNone(file_list, 'List of files should not be None') + + def test_hostname_no_tailing_slash(self): + self.assertEqual('5', self.client.webdav.hostname[-1]) + + +if __name__ == '__main__': + unittest.main() diff --git a/webdav3/connection.py b/webdav3/connection.py index 302212b..499ba23 100644 --- a/webdav3/connection.py +++ b/webdav3/connection.py @@ -50,6 +50,7 @@ class WebDAVSettings(ConnectionSettings): self.root = Urn(self.root).quote() if self.root else '' self.root = self.root.rstrip(Urn.separate) + self.hostname = self.hostname.rstrip(Urn.separate) def is_valid(self): if not self.hostname: diff --git a/webdav3/urn.py b/webdav3/urn.py index 6279de2..e78fa24 100644 --- a/webdav3/urn.py +++ b/webdav3/urn.py @@ -34,13 +34,11 @@ class Urn(object): return self._path def filename(self): - path_split = self._path.split(Urn.separate) name = path_split[-2] + Urn.separate if path_split[-1] == '' else path_split[-1] return unquote(name) def parent(self): - path_split = self._path.split(Urn.separate) nesting_level = self.nesting_level() parent_path_split = path_split[:nesting_level] From a3c75b7155662cbc0c63e202e20dfea2e8d76480 Mon Sep 17 00:00:00 2001 From: Alex Jordan Date: Mon, 6 Apr 2020 08:39:51 -0400 Subject: [PATCH 19/19] Fixes ezhov-evgeny#43 - local certificate options ignored If the webdav_cert_path and webdav_key_path options are set, they should be passed to the call to requests in Client.execute_request as the cert parameter. --- webdav3/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/webdav3/client.py b/webdav3/client.py index c88a172..9e495d1 100644 --- a/webdav3/client.py +++ b/webdav3/client.py @@ -217,6 +217,7 @@ class Client(object): auth=(self.webdav.login, self.webdav.password) if (not self.webdav.token and not self.session.auth) else None, headers=self.get_headers(action, headers_ext), timeout=self.timeout, + cert=(self.webdav.cert_path, self.webdav.key_path) if (self.webdav.cert_path and self.webdav.key_path) else None, data=data, stream=True, verify=self.verify