diff --git a/README.md b/README.md index 3406e29..b8f14fe 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ $ otag purge old-project/ ### Embedded Metadata -`scan` (`s`) reads tags that are embedded in files and adds them to their entries, in the given folder (default: current) and its subdirectories. `--replace` makes the found tags replace the entry instead, so tags removed in another app also disappear from otag; files without embedded tags keep their entry. +`scan` (`s`) reads tags that are embedded in files and adds them to their entries, in the given folder (default: current) and its subdirectories. Plugins that support folders, like `tagspaces`, also read the subfolders' tags. `--replace` makes the found tags replace the entry instead, so tags removed in another app also disappear from otag; files without embedded tags keep their entry. `write` (`w`) writes the entry of a file into the file itself, using the named plugin. It replaces what the plugin stored before, so tags removed with `otag remove` disappear from the file on the next `write`. `-c` (`--clear`) removes the plugin's tags from the file instead, e.g. after all tags were removed; it is never done implicitly, because a file without entry may also be one otag never tagged. @@ -108,6 +108,7 @@ $ otag write --clear id3 song.mp3 | `id3` | `.mp3` `.wav` `.flac` `.ogg` | The comment tag | | `image` | `.jpg` `.jpeg` `.png` `.gif` | An `otag:` comment of its own next to any other comments: a COM segment in JPEG (after the JFIF and EXIF segments; some apps only show a JPEG's last comment), an iTXt `Comment` chunk in PNG, a comment extension in GIF (turning a GIF87a into GIF89a). The image data stays unchanged | | `pdf` | `.pdf` | The keywords field | +| `tagspaces` | Every file type and folders | The JSON sidecar [TagSpaces](https://www.tagspaces.org) uses: `.ts/.json` next to a file, `.ts/tsm.json` inside a folder. Tags are written bare as in TagSpaces, projects and contexts keep their `+` and `@`; spaces in TagSpaces tags become `_`. Tags in a file name like `photo[vacation alps].jpg` are read but never written, since that would rename the file. The note and the sidecar's description, colors and other data are left untouched | | `office` | `.docx` `.xlsx` `.pptx` `.odt` `.ods` `.odp`, reading also `.doc` `.xls` `.ppt` | The description field | | `ebook` | `.epub` `.fb2` `.cbz` | One otag line where other apps don't look: a `` in the EPUB package file, a `` in FB2, the archive comment of CBZ (never overwriting another tool's comment); the book's own metadata is left untouched | | `archive` | `.zip` `.tar` `.gz` `.tgz` (so also `.tar.gz`) | An `otag:` comment that unpacking ignores: the archive comment of ZIP, the comment field of the gzip header, a pax global header at the start of a tar. Another tool's comment, like the commit id stored by `git archive`, is never replaced. Writing a tar or gzip file copies it once, the packed data stays unchanged | diff --git a/orvanitag/cmd.py b/orvanitag/cmd.py index de17911..6afd125 100644 --- a/orvanitag/cmd.py +++ b/orvanitag/cmd.py @@ -240,7 +240,7 @@ def parser() -> ArgumentParser: no_recursive(sub) sub = command('write', ['w'], 'write the entry of a file into the file itself', run_write) - sub.add_argument('plugin', help='archive, comment, ebook, frontmatter, id3, image, office or pdf') + sub.add_argument('plugin', help='archive, comment, ebook, frontmatter, id3, image, office, pdf or tagspaces') file_arg(sub) sub.add_argument('-c', '--clear', action='store_true', help="remove the plugin's tags from the file instead") diff --git a/orvanitag/orvani.py b/orvanitag/orvani.py index 923d418..9eac728 100644 --- a/orvanitag/orvani.py +++ b/orvanitag/orvani.py @@ -14,6 +14,7 @@ CONTEXT_FILE = '.otag' CONFIG_FILE = '.otag.toml' CACHE_FILE = '.otag.cache.json' SEPARATOR = ' --- ' # between file name and contexts, in context files and exports +ANY = '*' # in the TYPES of plugins that handle every file type DEFAULT_PLUGINS = [ 'frontmatter', @@ -23,7 +24,8 @@ DEFAULT_PLUGINS = [ 'id3', 'image', 'office', - 'pdf' + 'pdf', + 'tagspaces' ] def walk(path, recursive: bool=True): @@ -37,6 +39,13 @@ def walk(path, recursive: bool=True): break +def handles(plugin, suffixes: list, path: Path) -> bool: + """Whether a plugin handles a path with the file types enabled for it; folders need the plugin's FOLDERS flag""" + if path.is_dir(): + return getattr(plugin, 'FOLDERS', False) + return ANY in suffixes or path.suffix.lower() in suffixes + + def key(path) -> str: """The index and export form of a path: normalized, with forward slashes on every system""" return Path(path).as_posix() @@ -444,14 +453,16 @@ class OrvaniTag: def scan(self, path: str, recursive: bool=False, replace: bool=False): """ - Adds the tags embedded in the files of a folder to their entries. + Adds the tags embedded in the files and subfolders of a folder to their entries. With replace they replace the entries of files they were found in. Returns the found entries by path and the messages of failed plugin reads """ found, errors = {}, [] for folder, files in walk(path, recursive): - for file in files: + # Subfolders only, since the entry of the scanned folder itself is kept above it + folders = sorted(item.name for item in os.scandir(folder) if item.is_dir()) + for file in folders + files: try: entry = self.read_metadata(folder / file, replace) except PluginError as error: @@ -495,7 +506,7 @@ class OrvaniTag: if 'plugins' in config and plugin_name not in config['plugins']: continue configured = config.get(plugin_name, {}).get('types', []) - yield plugin_name, plugin, [suffix for suffix in configured if suffix in plugin.TYPES] or plugin.TYPES + yield plugin_name, plugin, [suffix for suffix in configured if suffix in plugin.TYPES or ANY in plugin.TYPES] or plugin.TYPES def read_metadata(self, filename: str, replace: bool=False) -> Entry: @@ -511,7 +522,7 @@ class OrvaniTag: found, errors = Entry(), [] if self.ENABLE_PLUGINS: for plugin_name, plugin, suffixes in self.__enabled(config): - if path.suffix.lower() not in suffixes or not hasattr(plugin, 'read'): + if not handles(plugin, suffixes, path) or not hasattr(plugin, 'read'): continue try: found.merge(plugin.read(path, config) or Entry()) @@ -552,7 +563,9 @@ class OrvaniTag: for plugin_name, plugin, suffixes in self.__enabled(config): if plugin.NAME != method: # select the reference plugin continue - if path.suffix.lower() not in suffixes: + if not handles(plugin, suffixes, path): + if path.is_dir(): + raise UnsupportedType(f"Plugin {plugin_name} can't write to folders.") configured = f' Its types are set in {CONFIG_FILE}.' if 'types' in config.get(plugin_name, {}) else '' raise UnsupportedType(f"Plugin {plugin_name} can only write to file types: {', '.join(suffixes)}.{configured}") if not hasattr(plugin, 'write'): diff --git a/orvanitag/plugins/__init__.py b/orvanitag/plugins/__init__.py index 35ede35..3c7a133 100644 --- a/orvanitag/plugins/__init__.py +++ b/orvanitag/plugins/__init__.py @@ -2,7 +2,8 @@ Plugins read and write tags embedded in files. A plugin is a module with: NAME the name used by `otag write NAME FILE` -TYPES the lowercase file suffixes it handles, e.g. ['.pdf'] +TYPES the lowercase file suffixes it handles, e.g. ['.pdf'], or ['*'] for every file type +FOLDERS optional, True if it also handles folders read(path: Path, config: dict) -> Entry | None returns the tags found in the file, or None write(path: Path, entry: Entry, config: dict) diff --git a/orvanitag/plugins/tagspaces.py b/orvanitag/plugins/tagspaces.py new file mode 100644 index 0000000..963ae90 --- /dev/null +++ b/orvanitag/plugins/tagspaces.py @@ -0,0 +1,84 @@ +""" +Tags as TagSpaces stores them: in a JSON sidecar `.ts/.json` next to a file, or `.ts/tsm.json` inside a folder. +TagSpaces tags are flat, so projects and contexts keep their `+` and `@` prefix, while tags are bare as usual there. +Tags in brackets at the end of a file name, `photo[vacation alps].jpg`, are read but never written, which would rename the file. +The sidecar's description and other data are left untouched. +""" +import json +import os +import re +import shutil +import time +import uuid +from orvanitag.entry import Entry, CONTEXTS, PREFIXES +from orvanitag.errors import UnsupportedFile + +NAME = 'tagspaces' +TYPES = ['*'] +FOLDERS = True +META = '.ts' +NAME_TAGS = re.compile(r'\[([^\[\]]*)\]$') + + +def sidecar(path): + return path / META / 'tsm.json' if path.is_dir() else path.parent / META / f'{path.name}.json' + + +def name_tags(path) -> list: + """The tags in brackets at the end of a file name""" + found = None if path.is_dir() else NAME_TAGS.search(path.stem) + return found.group(1).split() if found else [] + + +def word(title) -> str: + """A TagSpaces tag title as one otag word, without the `#` that tags do not have in TagSpaces""" + return '_'.join(str(title or '').split()).removeprefix('#') + + +def load(path) -> dict: + file = sidecar(path) + if not file.is_file(): + return {} + try: + data = json.loads(file.read_text(encoding='utf-8-sig')) + except ValueError as error: + raise UnsupportedFile(f'{file} is no valid JSON: {error}') from error + if not isinstance(data, dict): + raise UnsupportedFile(f'{file} is no TagSpaces sidecar') + return data + + +def tags(data: dict) -> list: + return [tag for tag in data.get('tags') or [] if isinstance(tag, dict)] + + +def read(path, config): + words = [word(tag.get('title')) for tag in tags(load(path))] + [word(title) for title in name_tags(path)] + words = [word if word[0] in PREFIXES else '#' + word for word in words if word] + if words: + return Entry.parse(words) + + +def write(path, entry, config): + file = sidecar(path) + exists = file.is_file() + if not entry and not exists: + return + data = load(path) if exists else {'id': uuid.uuid4().hex} + # Tags that stay keep their TagSpaces title and colors + known = {word(tag.get('title')): tag for tag in tags(data)} + in_name = {word(title) for title in name_tags(path)} + titles = [CONTEXTS[name] + value for name in ('projects', 'contexts') for value in getattr(entry, name)] + entry.tags + data['tags'] = [known.get(title) or {'title': title, 'type': 'sidecar'} for title in titles if title not in in_name] + if 'lastUpdated' in data: + data['lastUpdated'] = int(time.time() * 1000) + + file.parent.mkdir(exist_ok=True) + tmp = file.with_name(f'.{file.name}.tmp') + try: + tmp.write_text(json.dumps(data, ensure_ascii=False), encoding='utf-8') + if exists: + shutil.copymode(file, tmp) + os.replace(tmp, file) + finally: + tmp.unlink(missing_ok=True) diff --git a/tests/test_tagspaces.py b/tests/test_tagspaces.py new file mode 100644 index 0000000..21cde93 --- /dev/null +++ b/tests/test_tagspaces.py @@ -0,0 +1,117 @@ +import json +import sys + +import pytest + +import orvanitag.orvani as orvani +from orvanitag.cmd import main +from orvanitag.entry import Entry +from orvanitag.orvani import OrvaniTag +from orvanitag.errors import UnsupportedType + + +@pytest.fixture +def cwd(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + return tmp_path + + +def sidecar(cwd, name): + return json.loads((cwd / '.ts' / f'{name}.json').read_text()) + + +def test_sidecar_roundtrip(cwd): + (cwd / 'song.mp3').touch() + OrvaniTag().add('song.mp3', '+album', '@drums', '#lofi', 'a note') + OrvaniTag().write_metadata('song.mp3', 'tagspaces') + data = sidecar(cwd, 'song.mp3') + assert [tag['title'] for tag in data['tags']] == ['+album', '@drums', 'lofi'] + assert data['tags'][0]['type'] == 'sidecar' and len(data['id']) == 32 + + (cwd / orvani.CONTEXT_FILE).unlink() + assert OrvaniTag().scan('.') == ({'song.mp3': Entry(['album'], ['drums'], ['lofi'])}, []) + + +def test_existing_sidecar_keeps_its_data(cwd): + (cwd / 'a.heic').touch() + (cwd / '.ts').mkdir() + (cwd / '.ts' / 'a.heic.json').write_text(json.dumps({ + 'appName': 'TagSpaces', 'id': 'x1', 'description': '# Trip\n\nin *markdown*', 'lastUpdated': 1, + 'tags': [{'title': 'new york', 'color': '#ffcc24', 'textcolor': '#ffffff', 'type': 'sidecar'}, + {'title': '#gone', 'type': 'sidecar'}], + })) + assert OrvaniTag().read_metadata('a.heic') == Entry(tags=['new_york', 'gone']) + + OrvaniTag().remove('a.heic', '#gone') + OrvaniTag().add('a.heic', '@city') + OrvaniTag().write_metadata('a.heic', 'tagspaces') + data = sidecar(cwd, 'a.heic') + assert data['tags'] == [{'title': '@city', 'type': 'sidecar'}, + {'title': 'new york', 'color': '#ffcc24', 'textcolor': '#ffffff', 'type': 'sidecar'}] + assert data['description'] == '# Trip\n\nin *markdown*' and data['id'] == 'x1' and data['appName'] == 'TagSpaces' + assert data['lastUpdated'] > 1 + + +def test_folder_tags(cwd): + (cwd / 'sub' / 'deeper').mkdir(parents=True) + OrvaniTag().add('sub', '+trip') + OrvaniTag().write_metadata('sub', 'tagspaces') + assert json.loads((cwd / 'sub' / '.ts' / 'tsm.json').read_text())['tags'] == [{'title': '+trip', 'type': 'sidecar'}] + + (cwd / 'sub' / 'deeper' / '.ts').mkdir() + (cwd / 'sub' / 'deeper' / '.ts' / 'tsm.json').write_text('{"tags": [{"title": "deep"}]}') + (cwd / orvani.CONTEXT_FILE).unlink() + assert OrvaniTag().scan('sub') == ({'sub/deeper': Entry(tags=['deep'])}, []) # the scanned folder itself is skipped + assert OrvaniTag().scan('.', recursive=True)[0] == {'sub': Entry(['trip']), 'sub/deeper': Entry(tags=['deep'])} + assert (cwd / orvani.CONTEXT_FILE).read_text() == 'sub/ --- +trip\n' + + +def test_name_tags_are_read_and_not_repeated(cwd): + (cwd / 'IMG[vacation alps].heic').touch() + assert OrvaniTag().read_metadata('IMG[vacation alps].heic') == Entry(tags=['vacation', 'alps']) + OrvaniTag().add('IMG[vacation alps].heic', '#beach') + OrvaniTag().write_metadata('IMG[vacation alps].heic', 'tagspaces') + assert sidecar(cwd, 'IMG[vacation alps].heic')['tags'] == [{'title': 'beach', 'type': 'sidecar'}] + assert (cwd / 'IMG[vacation alps].heic').exists() + + +def test_clear_keeps_the_sidecar_data(cwd): + (cwd / 'a.txt').touch() + (cwd / 'b.txt').touch() + (cwd / '.ts').mkdir() + (cwd / '.ts' / 'a.txt.json').write_text('{"id": "x1", "description": "d", "tags": [{"title": "t"}]}') + OrvaniTag().write_metadata('a.txt', 'tagspaces', clear=True) + assert sidecar(cwd, 'a.txt') == {'id': 'x1', 'description': 'd', 'tags': []} + OrvaniTag().write_metadata('b.txt', 'tagspaces', clear=True) + assert not (cwd / '.ts' / 'b.txt.json').exists() + + +def test_broken_sidecar_is_a_scan_error(cwd): + (cwd / 'a.txt').touch() + (cwd / '.ts').mkdir() + (cwd / '.ts' / 'a.txt.json').write_text('{broken') + found, errors = OrvaniTag().scan('.') + assert found == {} and len(errors) == 1 and 'a.txt.json is no valid JSON' in errors[0] + + +def test_configured_types_narrow_plugins_for_any_type(cwd): + (cwd / orvani.CONFIG_FILE).write_text('[tagspaces]\ntypes = [".jpg"]\n') + for name in ['a.jpg', 'a.png']: + (cwd / name).touch() + OrvaniTag().add(name, '#x') + OrvaniTag().write_metadata('a.jpg', 'tagspaces') + with pytest.raises(UnsupportedType): + OrvaniTag().write_metadata('a.png', 'tagspaces') + + +def test_other_plugins_do_not_handle_folders(cwd, monkeypatch, capsys): + (cwd / 'sub.py').mkdir() + (cwd / 'sub.py' / 'x.py').write_text('# otag: #inner\n') + OrvaniTag().add('sub.py', '#x') + with pytest.raises(UnsupportedType, match="can't write to folders"): + OrvaniTag().write_metadata('sub.py', 'comment') + assert OrvaniTag().scan('.')[0] == {} + + monkeypatch.setattr(sys, 'argv', ['otag', 'w', 'tagspaces', 'sub.py/']) + main() + assert (cwd / 'sub.py' / '.ts' / 'tsm.json').exists()