From 965812b9f47b3595da16598c14406f2d9332fabb Mon Sep 17 00:00:00 2001 From: Randy Oswald Date: Mon, 14 Sep 2020 12:39:23 -0400 Subject: [PATCH 1/2] Recreate MR from KernpunktAnalytics --- tableaudocumentapi/__init__.py | 1 + tableaudocumentapi/datasource.py | 3 +- tableaudocumentapi/folder.py | 176 ++++++++++++++++++++++++++++ test/assets/folder_test.tds | 109 ++++++++++++++++++ test/test_folders.py | 192 +++++++++++++++++++++++++++++++ 5 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 tableaudocumentapi/folder.py create mode 100644 test/assets/folder_test.tds create mode 100644 test/test_folders.py diff --git a/tableaudocumentapi/__init__.py b/tableaudocumentapi/__init__.py index 6a10f6f..5a4d151 100644 --- a/tableaudocumentapi/__init__.py +++ b/tableaudocumentapi/__init__.py @@ -1,4 +1,5 @@ from .field import Field +from .folder import Folder, FolderItem from .connection import Connection from .datasource import Datasource, ConnectionParser from .workbook import Workbook diff --git a/tableaudocumentapi/datasource.py b/tableaudocumentapi/datasource.py index 418dc53..f83d3d3 100644 --- a/tableaudocumentapi/datasource.py +++ b/tableaudocumentapi/datasource.py @@ -5,7 +5,7 @@ from uuid import uuid4 from tableaudocumentapi import Connection, xfile -from tableaudocumentapi import Field +from tableaudocumentapi import Field, Folder from tableaudocumentapi.multilookup_dict import MultiLookupDict from tableaudocumentapi.xfile import xml_open @@ -137,6 +137,7 @@ def __init__(self, dsxml, filename=None): self._datasourceXML, version=self._version) self._connections = self._connection_parser.get_connections() self._fields = None + self._folders = None @classmethod def from_file(cls, filename): diff --git a/tableaudocumentapi/folder.py b/tableaudocumentapi/folder.py new file mode 100644 index 0000000..d645351 --- /dev/null +++ b/tableaudocumentapi/folder.py @@ -0,0 +1,176 @@ +import xml.etree.ElementTree as ET +from functools import wraps + +from tableaudocumentapi import Field + +######## +# This is needed in order to determine if something is a string or not. It is necessary because +# of differences between python2 (basestring) and python3 (str). If python2 support is every +# dropped, remove this and change the basestring references below to str +try: + basestring +except NameError: # pragma: no cover + basestring = str + + +class AlreadyMemberOfThisFolderException(Exception): + pass + + +class MemberOfMultipleFoldersException(Exception): + pass + + +def argument_is_one_of(*allowed_values): + def property_type_decorator(func): + @wraps(func) + def wrapper(self, value): + if value not in allowed_values: + error = "Invalid argument: {0}. {1} must be one of {2}." + msg = error.format(value, func.__name__, allowed_values) + raise ValueError(error) + return func(self, value) + return wrapper + return property_type_decorator + + +class FolderItem(object): + """ FolderItems belong to Folders and describe the Field-Objects + that belong to a folder + """ + + def __init__(self, name, _type): + self.name = name + self.type = _type + + @classmethod + def from_xml(cls, xml): + return cls(xml.get('name', None), xml.get('type', None)) + + @classmethod + def from_field(cls, field): + return cls(field.id, 'field') + + +class Folder(object): + """ This class represents a folder in a Datasource. + + Folders have a name, a role (dimensions or measures) and contain Items + """ + + def __init__(self, datasource, xml): + self._datasource = datasource + self._xml = xml + self.name = self._xml.get('name', None) + self.role = self._xml.get('role', None) + folder_item_xml = self._xml.findall('folder-item') + self._folder_items = [FolderItem.from_xml(xml) for xml in folder_item_xml] + + # Alternative constructors + + @classmethod + def all_folders_from_datasource(cls, datasource): + folders_xml = datasource._datasourceTree.findall('.//folder') + return [cls(datasource, xml) for xml in folders_xml] + + @classmethod + def from_name_and_role(cls, name, role, parent_datasource): + """Creates a new folder with a given name and a given role. + """ + attributes = { + 'name': name, + 'role': role + } + xml = ET.Element('folder', attrib=attributes) + return cls(parent_datasource, xml) + + # Properties + + @property + def folder_items(self): + return self._folder_items + + @property + def name(self): + return self._name + + @name.setter + def name(self, name): + self._name = name + self._xml.set('name', name) + + @property + def role(self): + return self._role + + @property + def xml(self): + return self._xml + + @role.setter + @argument_is_one_of('dimensions', 'measures') + def role(self, role): + self._role = role + self._xml.set('role', role) + + # Functions that deal with folder-items + + def add_field(self, field): + """ Adds a field to this folder + """ + if not isinstance(field, Field): + msg = 'Can only add Fields to Folders, not {}' + raise ValueError(msg.format(type(field))) + if self.has_item(field): + raise AlreadyMemberOfThisFolderException(field) + if any(f.has_item(field) for f in self._datasource.folders.values()): + raise MemberOfMultipleFoldersException(field) + + self._add_field(field) + + def _add_field(self, field): + """ Internal function to add a field + """ + folder_item = FolderItem.from_field(field) + self._folder_items.append(folder_item) + name, _type = folder_item.name, folder_item.type + ET.SubElement(self._xml, 'folder-item', {'name': name, 'type': _type}) + + def remove_field(self, field): + """ Removes a field from this folder + """ + if not isinstance(field, Field): + msg = 'Can only remove Fields from Folders, not {}' + raise ValueError(msg.format(type(field))) + if not self.has_item(field): + raise ValueError('This field is not a member of the folder') + + self._remove_field(field) + + def _remove_field(self, field): + """ Internal function to remove field + """ + # remove from the data structure + folder_items = filter(lambda f: f.name == field.id, self.folder_items) + folder_item = list(folder_items)[0] + self.folder_items.remove(folder_item) + + # remove from xml + xml_elem = self.xml.find("folder-item[@name='{}']".format(field.id)) + self.xml.remove(xml_elem) + + # Utility functions + + def has_item(self, item): + """ Returns True if the given item is a FolderItem of this Folder. + Item may be String, Field or FolderItem + """ + if isinstance(item, FolderItem): + return item in self.folder_items + elif isinstance(item, Field): + return item.id in map(lambda fi: fi.name, self.folder_items) + elif isinstance(item, basestring): + return item in map(lambda fi: fi.name, self.folder_items) + else: + msg = 'Argument must be either String or FolderItem, not {}' + raise ValueError(msg.format(type(item))) \ No newline at end of file diff --git a/test/assets/folder_test.tds b/test/assets/folder_test.tds new file mode 100644 index 0000000..8e8f4e2 --- /dev/null +++ b/test/assets/folder_test.tds @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + name + 130 + [name] + [my_data] + name + 1 + string + Count + 8190 + true + true + + + "SQL_WLONGVARCHAR" + "SQL_C_WCHAR" + + + + typ + 130 + [typ] + [my_data] + typ + 2 + string + Count + 8190 + true + true + + + "SQL_WLONGVARCHAR" + "SQL_C_WCHAR" + + + + amount + 3 + [amount] + [my_data] + amount + 3 + integer + Sum + 10 + true + + "SQL_INTEGER" + "SQL_C_SLONG" + + + + price + 5 + [price] + [my_data] + price + 4 + real + Sum + 17 + true + + "SQL_FLOAT" + "SQL_C_DOUBLE" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/test_folders.py b/test/test_folders.py new file mode 100644 index 0000000..a0b75b9 --- /dev/null +++ b/test/test_folders.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- + +import os.path +import unittest +import xml.etree.ElementTree as ET + +from tableaudocumentapi import Datasource +from tableaudocumentapi.folder import (AlreadyMemberOfThisFolderException, + Folder, FolderItem, + MemberOfMultipleFoldersException) + +TEST_ASSET_DIR = os.path.join( + os.path.dirname(__file__), + 'assets' +) +TEST_TDS_FILE = os.path.join( + TEST_ASSET_DIR, + 'folder_test.tds' +) +TEST_TDS_TEMP_FILE = os.path.join( + TEST_ASSET_DIR, + 'folder_test_temp.tds' +) + + +class TestFolders(unittest.TestCase): + + def setUp(self): + self.tds = Datasource.from_file(TEST_TDS_FILE) + + def current_hash(self): + """ Return a hash of the current state of the XML. + + Allows us to easily identify whether the underlying XML-structure + of a TDS-file has actually changed. Avoids false positives if, + for example, a fields value has changed but the XML hasn't. + """ + return hash(ET.tostring(self.tds._datasourceTree.getroot())) + + def test_get_existing_folders(self): + """ Test whether pre-existing folders in a TDS file are recognized. + + The test-TDS-file already contains a folder named "MyTestFolder" + This folder already contains a field named "[price]" + """ + folder = self.tds.folders["MyTestFolder"] + self.assertIsNotNone(folder) + self.assertTrue(folder.has_item('[price]')) + + def test_fields_in_folders(self): + """ Test whether folder-items of a folder can be extracted. + Allowed types are: FolderItem, String and Field + """ + folder = self.tds.folders["MyTestFolder"] + + # check String + self.assertTrue(folder.has_item('[price]')) + self.assertFalse(folder.has_item('[bananas]')) + + # check FolderItem + self.assertTrue(folder.has_item(folder.folder_items[0])) + + # check Field + field = self.tds.fields['[price]'] + self.assertTrue(folder.has_item(field)) + + # should fail for invalid types + with self.assertRaises(ValueError): + folder.has_item(404) + + def test_add_to_folder(self): + """ A folder should be able to accept new fields. + + The test-TDS-file already contains a folder named "MyTestFolder" + This folder already contains a single field named "[price]" + """ + field_already_in_folder = '[price]' + field_already_in_other_folder = '[name]' + field_not_in_a_folder = '[typ]' + + # if a field is already in a folder + field = self.tds.fields[field_already_in_folder] + self.assertIsNotNone(field) + folder = self.tds.folders["MyTestFolder"] + with self.assertRaises(AlreadyMemberOfThisFolderException): + folder.add_field(field) + + # if a field is already in another folder + field = self.tds.fields[field_already_in_other_folder] + self.assertIsNotNone(field) + folder = self.tds.folders["MyTestFolder"] + with self.assertRaises(MemberOfMultipleFoldersException): + folder.add_field(field) + + # a field that can actually be added + field = self.tds.fields[field_not_in_a_folder] + self.assertIsNotNone(field) + folder = self.tds.folders["MyTestFolder"] + folder.add_field(field) + self.assertTrue(folder.has_item(field)) + + # check persistence + self.tds.save_as(TEST_TDS_TEMP_FILE) + persisted_tds = Datasource.from_file(TEST_TDS_TEMP_FILE) + persisted_folder = persisted_tds.folders['MyTestFolder'] + self.assertTrue(persisted_folder.has_item(field_not_in_a_folder)) + + def test_remove_from_folder(self): + """ A folder should be able to delete fields. + + The test-TDS-file already contains a folder named "MyTestFolder" + This folder already contains a single field named "[price]" + """ + field_already_in_folder = '[price]' + field_not_in_a_folder = '[typ]' + + # if a field is not in a folder + field = self.tds.fields[field_not_in_a_folder] + self.assertIsNotNone(field) + folder = self.tds.folders["MyTestFolder"] + with self.assertRaises(ValueError): + folder.remove_field(field) + + # a field that can actually be removed + field = self.tds.fields[field_already_in_folder] + self.assertIsNotNone(field) + folder = self.tds.folders["MyTestFolder"] + folder.remove_field(field) + self.assertFalse(folder.has_item(field)) + + # check persistence + self.tds.save_as(TEST_TDS_TEMP_FILE) + persisted_tds = Datasource.from_file(TEST_TDS_TEMP_FILE) + persisted_folder = persisted_tds.folders['MyTestFolder'] + self.assertFalse(persisted_folder.has_item(field_already_in_folder)) + + def test_change_attributes(self): + name = u"蚵仔煎" + folder = self.tds.folders["MyTestFolder"] + folder.name = name + folder.role = "measures" + + self.tds.save_as(TEST_TDS_TEMP_FILE) + persisted_tds = Datasource.from_file(TEST_TDS_TEMP_FILE) + persisted_folder = persisted_tds.folders[name] + self.assertIsNotNone(persisted_folder) + self.assertEqual(persisted_folder.name, name) + self.assertEqual(persisted_folder.role, "measures") + + def test_add_new_folder(self): + """ The test-TDS-file should allow new folders to be added. + """ + folder_name = u"蚵仔煎" + folder_role = "dimensions" + field_not_in_a_folder = '[typ]' + + new_folder = self.tds.add_folder(folder_name, folder_role) + + # add a new field to that folder + field = self.tds.fields[field_not_in_a_folder] + new_folder.add_field(field) + + # check if the folder is there + folder = self.tds.folders[folder_name] + self.assertIsNotNone(folder) + + # test persistence + self.tds.save_as(TEST_TDS_TEMP_FILE) + persisted_tds = Datasource.from_file(TEST_TDS_TEMP_FILE) + persisted_folder = persisted_tds.folders[folder_name] + self.assertIsNotNone(persisted_folder) + self.assertTrue(persisted_folder.has_item(field)) + + def test_add_folder_fails(self): + """ The test-TDS-file should NOT allow duplicated folders to be added. + + The test-TDS-file already contains a folder named "testfolder" + This folder already contains a single field named "[price]" + """ + folder_name = "another_testfolder" + folder_role = "dimensions" + field_not_in_a_folder = '[typ]' + + with self.assertRaises(ValueError): + self.tds.add_folder(folder_name, "dimenzzionss") + + with self.assertRaises(ValueError): + self.tds.add_folder("MyTestFolder", folder_role) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From e646b95a50bf78a43adf78f703d23ae5ad045e0b Mon Sep 17 00:00:00 2001 From: Randy Oswald Date: Mon, 14 Sep 2020 12:40:18 -0400 Subject: [PATCH 2/2] Add in Folder section from Merge Request. --- tableaudocumentapi/datasource.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tableaudocumentapi/datasource.py b/tableaudocumentapi/datasource.py index f83d3d3..7a5594b 100644 --- a/tableaudocumentapi/datasource.py +++ b/tableaudocumentapi/datasource.py @@ -246,3 +246,33 @@ def _get_metadata_objects(self): def _get_column_objects(self): return [_column_object_from_column_xml(self._datasourceTree, xml) for xml in self._datasourceTree.findall('.//column')] + + ########### + # folders + ########### + @property + def folders(self): + if not self._folders: + self._refresh_folders() + return self._folders + + def _refresh_folders(self): + folders = Folder.all_folders_from_datasource(self) + self._folders = {f.name: f for f in folders} + + def add_folder(self, name, role): + """ Adds a new, empty folder to the datasource and returns it. + Will fail if another folder of the same name already exists, + or if no valid role (dimensions or measures) is provided. + """ + + if name in self.folders.keys(): + raise ValueError('Folder names must be unique') + + # Create the folder object + parent_datasource = self # The parent of the new folder is the current datasource + folder = Folder.from_name_and_role(name, role, parent_datasource) + # Add the folder xml to the datasources xml + self._datasourceXML.append(folder.xml) + self._refresh_folders() + return folder \ No newline at end of file