Upload 9 files

language_tool_python/__main__.py

@@ -0,0 +1,178 @@
+"""LanguageTool command line."""
+
+import argparse
+import locale
+import re
+import sys
+
+from .server import LanguageTool
+from .utils import LanguageToolError
+
+import pkg_resources
+__version__ = pkg_resources.require("language_tool_python")[0].version
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=__doc__.strip() if __doc__ else None,
+        prog='language_tool_python')
+    parser.add_argument('files', nargs='+',
+                        help='plain text file or "-" for stdin')
+    parser.add_argument('-c', '--encoding',
+                        help='input encoding')
+    parser.add_argument('-l', '--language', metavar='CODE',
+                        help='language code of the input or "auto"')
+    parser.add_argument('-m', '--mother-tongue', metavar='CODE',
+                        help='language code of your first language')
+    parser.add_argument('-d', '--disable', metavar='RULES', type=get_rules,
+                        action=RulesAction, default=set(),
+                        help='list of rule IDs to be disabled')
+    parser.add_argument('-e', '--enable', metavar='RULES', type=get_rules,
+                        action=RulesAction, default=set(),
+                        help='list of rule IDs to be enabled')
+    parser.add_argument('--enabled-only', action='store_true',
+                        help='disable all rules except those specified in '
+                             '--enable')
+    parser.add_argument(
+        '--version', action='version',
+        version='%(prog)s {}'.format(__version__),
+        help='show version')
+    parser.add_argument('-a', '--apply', action='store_true',
+                        help='automatically apply suggestions if available')
+    parser.add_argument('-s', '--spell-check-off', dest='spell_check',
+                        action='store_false',
+                        help='disable spell-checking rules')
+    parser.add_argument('--ignore-lines',
+                        help='ignore lines that match this regular expression')
+    parser.add_argument('--remote-host',
+                        help='hostname of the remote LanguageTool server')
+    parser.add_argument('--remote-port',
+                        help='port of the remote LanguageTool server')
+
+    args = parser.parse_args()
+
+    if args.enabled_only:
+        if args.disable:
+            parser.error('--enabled-only cannot be used with --disable')
+
+        if not args.enable:
+            parser.error('--enabled-only requires --enable')
+
+    return args
+
+
+class RulesAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        getattr(namespace, self.dest).update(values)
+
+
+def get_rules(rules: str) -> set:
+    return {rule.upper() for rule in re.findall(r"[\w\-]+", rules)}
+
+
+def get_text(filename, encoding, ignore):
+    with open(filename, encoding=encoding) as f:
+        text = ''.join('\n' if (ignore and re.match(ignore, line)) else line
+                       for line in f.readlines())
+    return text
+
+
+def print_unicode(text):
+    """Print in a portable manner."""
+    if sys.version_info[0] < 3:
+        text = text.encode('utf-8')
+
+    print(text)
+
+
+def main():
+    args = parse_args()
+
+    status = 0
+
+    for filename in args.files:
+        if len(args.files) > 1:
+            print(filename, file=sys.stderr)
+
+        if filename == '-':
+            filename = sys.stdin.fileno()
+            encoding = args.encoding or (
+                sys.stdin.encoding if sys.stdin.isatty()
+                else locale.getpreferredencoding()
+            )
+        else:
+            encoding = args.encoding or 'utf-8'
+
+        remote_server = None
+        if args.remote_host is not None:
+            remote_server = args.remote_host
+            if args.remote_port is not None:
+                remote_server += ':{}'.format(args.remote_port)
+        lang_tool = LanguageTool(
+            motherTongue=args.mother_tongue,
+            remote_server=remote_server,
+        )
+        guess_language = None
+
+        try:
+            text = get_text(filename, encoding, ignore=args.ignore_lines)
+        except UnicodeError as exception:
+            print('{}: {}'.format(filename, exception), file=sys.stderr)
+            continue
+
+        if args.language:
+            if args.language.lower() == 'auto':
+                try:
+                    from guess_language import guess_language
+                except ImportError:
+                    print('guess_language is unavailable.', file=sys.stderr)
+                    return 1
+                else:
+                    language = guess_language(text)
+                    print('Detected language: {}'.format(language),
+                          file=sys.stderr)
+                    if not language:
+                        return 1
+                    lang_tool.language = language
+            else:
+                lang_tool.language = args.language
+
+        if not args.spell_check:
+            lang_tool.disable_spellchecking()
+
+        lang_tool.disabled_rules.update(args.disable)
+        lang_tool.enabled_rules.update(args.enable)
+        lang_tool.enabled_rules_only = args.enabled_only
+
+        try:
+            if args.apply:
+                print_unicode(lang_tool.correct(text))
+            else:
+                for match in lang_tool.check(text):
+                    rule_id = match.ruleId
+
+                    replacement_text = ', '.join(
+                        "'{}'".format(word)
+                        for word in match.replacements).strip()
+
+                    message = match.message
+
+                    # Messages that end with punctuation already include the
+                    # suggestion.
+                    if replacement_text and not message.endswith(('.', '?')):
+                        message += '; suggestions: ' + replacement_text
+
+                    print_unicode('{}: {}: {}'.format(
+                        filename,
+                        rule_id,
+                        message))
+
+                    status = 2
+        except LanguageToolError as exception:
+            print('{}: {}'.format(filename, exception), file=sys.stderr)
+            continue
+
+    return status
+
+
+sys.exit(main())

language_tool_python/config_file.py

@@ -0,0 +1,96 @@
+from typing import Any, Dict
+
+import atexit
+import os
+import tempfile
+
+ALLOWED_CONFIG_KEYS = { 
+    'maxTextLength', 'maxTextHardLength', 'secretTokenKey', 'maxCheckTimeMillis', 'maxErrorsPerWordRate',
+    'maxSpellingSuggestions', 'maxCheckThreads', 'cacheSize', 'cacheTTLSeconds', 'cacheSize', 'requestLimit',
+    'requestLimitInBytes', 'timeoutRequestLimit', 'requestLimitPeriodInSeconds', 'languageModel',
+    'word2vecModel', 'fasttextModel', 'fasttextBinary', 'maxWorkQueueSize', 'rulesFile', 'warmUp',
+    'blockedReferrers' 'premiumOnly', 'disabledRuleIds', 'pipelineCaching', 'maxPipelinePoolSize',
+    'pipelineCaching', 'pipelineExpireTimeInSeconds', 'pipelinePrewarming'
+}
+class LanguageToolConfig:
+    config: Dict[str, Any]
+    path: str
+    def __init__(self, config: Dict[str, Any]):
+        assert set(config.keys()) <= ALLOWED_CONFIG_KEYS, f"unexpected keys in config: {set(config.keys()) - ALLOWED_CONFIG_KEYS}"
+        assert len(config), "config cannot be empty"
+        self.config = config
+        self.path = self._create_temp_file()
+    
+    def _create_temp_file(self) -> str:
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+
+        # WRite key=value entries as lines in temporary file.
+        for key, value in self.config.items():
+            next_line = f'{key}={value}\n'
+            tmp_file.write(next_line.encode())
+        tmp_file.close()
+
+        # Remove file when program exits.
+        atexit.register(lambda: os.unlink(tmp_file.name))
+
+        return tmp_file.name
+
+
+
+"""
+❯ /usr/bin/java -cp /Users/johnmorris/.cache/language_tool_python/LanguageTool-5.6/languagetool-server.jar org.languagetool.server.HTTPServer --help
+Usage: HTTPServer [--config propertyFile] [--port|-p port] [--public]
+  --config FILE  a Java property file (one key=value entry per line) with values for:
+                 'maxTextLength' - maximum text length, longer texts will cause an error (optional)
+                 'maxTextHardLength' - maximum text length, applies even to users with a special secret 'token' parameter (optional)
+                 'secretTokenKey' - secret JWT token key, if set by user and valid, maxTextLength can be increased by the user (optional)
+                 'maxCheckTimeMillis' - maximum time in milliseconds allowed per check (optional)
+                 'maxErrorsPerWordRate' - checking will stop with error if there are more rules matches per word (optional)
+                 'maxSpellingSuggestions' - only this many spelling errors will have suggestions for performance reasons (optional,
+                                            affects Hunspell-based languages only)
+                 'maxCheckThreads' - maximum number of threads working in parallel (optional)
+                 'cacheSize' - size of internal cache in number of sentences (optional, default: 0)
+                 'cacheTTLSeconds' - how many seconds sentences are kept in cache (optional, default: 300 if 'cacheSize' is set)
+                 'requestLimit' - maximum number of requests per requestLimitPeriodInSeconds (optional)
+                 'requestLimitInBytes' - maximum aggregated size of requests per requestLimitPeriodInSeconds (optional)
+                 'timeoutRequestLimit' - maximum number of timeout request (optional)
+                 'requestLimitPeriodInSeconds' - time period to which requestLimit and timeoutRequestLimit applies (optional)
+                 'languageModel' - a directory with '1grams', '2grams', '3grams' sub directories which contain a Lucene index
+                                   each with ngram occurrence counts; activates the confusion rule if supported (optional)
+                 'word2vecModel' - a directory with word2vec data (optional), see
+                  https://github.com/languagetool-org/languagetool/blob/master/languagetool-standalone/CHANGES.md#word2vec
+                 'fasttextModel' - a model file for better language detection (optional), see
+                                   https://fasttext.cc/docs/en/language-identification.html
+                 'fasttextBinary' - compiled fasttext executable for language detection (optional), see
+                                    https://fasttext.cc/docs/en/support.html
+                 'maxWorkQueueSize' - reject request if request queue gets larger than this (optional)
+                 'rulesFile' - a file containing rules configuration, such as .langugagetool.cfg (optional)
+                 'warmUp' - set to 'true' to warm up server at start, i.e. run a short check with all languages (optional)
+                 'blockedReferrers' - a comma-separated list of HTTP referrers (and 'Origin' headers) that are blocked and will not be served (optional)
+                 'premiumOnly' - activate only the premium rules (optional)
+                 'disabledRuleIds' - a comma-separated list of rule ids that are turned off for this server (optional)
+                 'pipelineCaching' - set to 'true' to enable caching of internal pipelines to improve performance
+                 'maxPipelinePoolSize' - cache size if 'pipelineCaching' is set
+                 'pipelineExpireTimeInSeconds' - time after which pipeline cache items expire
+                 'pipelinePrewarming' - set to 'true' to fill pipeline cache on start (can slow down start a lot)
+                 Spellcheck-only languages: You can add simple spellcheck-only support for languages that LT doesn't
+                                            support by defining two optional properties:
+                   'lang-xx' - set name of the language, use language code instead of 'xx', e.g. lang-tr=Turkish
+                   'lang-xx-dictPath' - absolute path to the hunspell .dic file, use language code instead of 'xx', e.g.
+                                        lang-tr-dictPath=/path/to/tr.dic. Note that the same directory also needs to
+                                        contain a common_words.txt file with the most common 10,000 words (used for better language detection)
+  --port, -p PRT   port to bind to, defaults to 8081 if not specified
+  --public         allow this server process to be connected from anywhere; if not set,
+                   it can only be connected from the computer it was started on
+  --allow-origin [ORIGIN] set the Access-Control-Allow-Origin header in the HTTP response,
+                         used for direct (non-proxy) JavaScript-based access from browsers.
+                         Example: --allow-origin "https://my-website.org"
+                         Don't set a parameter for `*`, i.e. access from all websites.
+  --verbose, -v    in case of exceptions, log the input text (up to 500 characters)
+  --languageModel  a directory with '1grams', '2grams', '3grams' sub directories (per language)
+                         which contain a Lucene index (optional, overwrites 'languageModel'
+                         parameter in properties files)
+  --word2vecModel  a directory with word2vec data (optional), see
+                   https://github.com/languagetool-org/languagetool/blob/master/languagetool-standalone/CHANGES.md#word2vec
+  --premiumAlways  activate the premium rules even when user has no username/password - useful for API servers
+"""

language_tool_python/console_mode.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""Write to stdout without causing UnicodeEncodeError."""
+
+import sys
+
+
+if (getattr(sys.stdout, 'errors', '') == 'strict' and
+        not getattr(sys.stdout, 'encoding', '').lower().startswith('utf')):
+    try:
+        import translit
+        sys.stdout = translit.StreamFilter(sys.stdout)
+    except ImportError:
+        import codecs
+        import unicodedata
+        import warnings
+
+        TRANSLIT_MAP = {
+            0x2018: "'",
+            0x2019: "'",
+            0x201c: '"',
+            0x201d: '"',
+        }
+
+        def simplify(s):
+            s = s.translate(TRANSLIT_MAP)
+            return ''.join([c for c in unicodedata.normalize('NFKD', s)
+                            if not unicodedata.combining(c)])
+
+        def simple_translit_error_handler(error):
+            if not isinstance(error, UnicodeEncodeError):
+                raise error
+            chunk = error.object[error.start:error.end]
+            repl = simplify(chunk)
+            repl = (repl.encode(error.encoding, 'backslashreplace')
+                    .decode(error.encoding))
+            return repl, error.end
+
+        class SimpleTranslitStreamFilter:
+
+            """Filter a stream through simple transliteration."""
+            errors = 'simple_translit'
+
+            def __init__(self, target):
+                self.target = target
+
+            def __getattr__(self, name):
+                return getattr(self.target, name)
+
+            def write(self, s):
+                self.target.write(self.downgrade(s))
+
+            def writelines(self, lines):
+                self.target.writelines(
+                    [self.downgrade(line) for line in lines])
+
+            def downgrade(self, s):
+                return (s.encode(self.target.encoding, self.errors)
+                        .decode(self.target.encoding))
+
+        codecs.register_error(SimpleTranslitStreamFilter.errors,
+                              simple_translit_error_handler)
+        sys.stdout = SimpleTranslitStreamFilter(sys.stdout)
+        warnings.warn('translit is unavailable', ImportWarning)

language_tool_python/download_lt.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Download latest LanguageTool distribution."""
+
+import logging
+import os
+import re
+import requests
+import subprocess
+import sys
+import tempfile
+import tqdm
+from typing import Optional
+import zipfile
+
+from distutils.spawn import find_executable
+from urllib.parse import urljoin
+from .utils import (
+    find_existing_language_tool_downloads,
+    get_language_tool_download_path,
+    LTP_JAR_DIR_PATH_ENV_VAR
+)
+
+# Create logger for this file.
+logging.basicConfig(format='%(message)s')
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+# Get download host from environment or default.
+BASE_URL = os.environ.get('LTP_DOWNLOAD_HOST', 'https://www.languagetool.org/download/')
+FILENAME = 'LanguageTool-{version}.zip'
+
+LTP_DOWNLOAD_VERSION = '6.4'
+
+JAVA_VERSION_REGEX = re.compile(
+    r'^(?:java|openjdk) version "(?P<major1>\d+)(|\.(?P<major2>\d+)\.[^"]+)"',
+    re.MULTILINE)
+
+# Updated for later versions of java
+JAVA_VERSION_REGEX_UPDATED = re.compile(
+    r'^(?:java|openjdk) [version ]?(?P<major1>\d+)\.(?P<major2>\d+)',
+    re.MULTILINE)
+
+
+def parse_java_version(version_text):
+    """Return Java version (major1, major2).
+
+    >>> parse_java_version('''java version "1.6.0_65"
+    ... Java(TM) SE Runtime Environment (build 1.6.0_65-b14-462-11M4609)
+    ... Java HotSpot(TM) 64-Bit Server VM (build 20.65-b04-462, mixed mode))
+    ... ''')
+    (1, 6)
+
+    >>> parse_java_version('''
+    ... openjdk version "1.8.0_60"
+    ... OpenJDK Runtime Environment (build 1.8.0_60-b27)
+    ... OpenJDK 64-Bit Server VM (build 25.60-b23, mixed mode))
+    ... ''')
+    (1, 8)
+
+    """
+    match = (
+        re.search(JAVA_VERSION_REGEX, version_text)
+        or re.search(JAVA_VERSION_REGEX_UPDATED, version_text)
+    )
+    if not match:
+        raise SystemExit(
+            'Could not parse Java version from """{}""".'.format(version_text))
+    major1 = int(match.group('major1'))
+    major2 = int(match.group('major2')) if match.group('major2') else 0
+    return (major1, major2)
+
+
+def confirm_java_compatibility():
+    """ Confirms Java major version >= 8. """
+    java_path = find_executable('java')
+    if not java_path:
+        raise ModuleNotFoundError(
+            'No java install detected. '
+            'Please install java to use language-tool-python.'
+        )
+
+    output = subprocess.check_output([java_path, '-version'],
+                                     stderr=subprocess.STDOUT,
+                                     universal_newlines=True)
+
+    major_version, minor_version = parse_java_version(output)
+    # Some installs of java show the version number like `14.0.1`
+    # and others show `1.14.0.1`
+    # (with a leading 1). We want to support both,
+    # as long as the major version is >= 8.
+    # (See softwareengineering.stackexchange.com/questions/175075/why-is-java-version-1-x-referred-to-as-java-x)
+    if major_version == 1 and minor_version >= 8:
+        return True
+    elif major_version >= 8:
+        return True
+    else:
+        raise SystemError('Detected java {}.{}. LanguageTool requires Java >= 8.'.format(major_version, minor_version))
+
+
+def get_common_prefix(z):
+    """Get common directory in a zip file if any."""
+    name_list = z.namelist()
+    if name_list and all(n.startswith(name_list[0]) for n in name_list[1:]):
+        return name_list[0]
+    return None
+
+
+def http_get(url, out_file, proxies=None):
+    """ Get contents of a URL and save to a file.
+    """
+    req = requests.get(url, stream=True, proxies=proxies)
+    content_length = req.headers.get('Content-Length')
+    total = int(content_length) if content_length is not None else None
+    if req.status_code == 403:  # Not found on AWS
+        raise Exception('Could not find at URL {}.'.format(url))
+    progress = tqdm.tqdm(unit="B", unit_scale=True, total=total,
+                         desc=f'Downloading LanguageTool {LTP_DOWNLOAD_VERSION}')
+    for chunk in req.iter_content(chunk_size=1024):
+        if chunk:  # filter out keep-alive new chunks
+            progress.update(len(chunk))
+            out_file.write(chunk)
+    progress.close()
+
+
+def unzip_file(temp_file, directory_to_extract_to):
+    """ Unzips a .zip file to folder path. """
+    logger.info(
+        'Unzipping {} to {}.'.format(temp_file.name, directory_to_extract_to)
+    )
+    with zipfile.ZipFile(temp_file.name, 'r') as zip_ref:
+        zip_ref.extractall(directory_to_extract_to)
+
+
+def download_zip(url, directory):
+    """ Downloads and unzips zip file from `url` to `directory`. """
+    # Download file.
+    downloaded_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False)
+    http_get(url, downloaded_file)
+    # Close the file so we can extract it.
+    downloaded_file.close()
+    # Extract zip file to path.
+    unzip_file(downloaded_file, directory)
+    # Remove the temporary file.
+    os.remove(downloaded_file.name)
+    # Tell the user the download path.
+    logger.info('Downloaded {} to {}.'.format(url, directory))
+
+
+def download_lt(language_tool_version: Optional[str] = LTP_DOWNLOAD_VERSION):
+    confirm_java_compatibility()
+
+    download_folder = get_language_tool_download_path()
+
+    # Use the env var to the jar directory if it is defined
+    # otherwise look in the download directory
+    if os.environ.get(LTP_JAR_DIR_PATH_ENV_VAR):
+        return
+
+    # Make download path, if it doesn't exist.
+    os.makedirs(download_folder, exist_ok=True)
+
+    assert os.path.isdir(download_folder)
+    old_path_list = find_existing_language_tool_downloads(download_folder)
+
+    if language_tool_version:
+        version = language_tool_version
+        filename = FILENAME.format(version=version)
+        language_tool_download_url = urljoin(BASE_URL, filename)
+        dirname, _ = os.path.splitext(filename)
+        extract_path = os.path.join(download_folder, dirname)
+
+        if extract_path in old_path_list:
+            return
+        download_zip(language_tool_download_url, download_folder)
+
+
+if __name__ == '__main__':
+    sys.exit(download_lt())

language_tool_python/language_tag.py

@@ -0,0 +1,38 @@
+import re
+
+from functools import total_ordering
+
+@total_ordering
+class LanguageTag:
+    """Language tag supported by LanguageTool."""
+    _LANGUAGE_RE = re.compile(r"^([a-z]{2,3})(?:[_-]([a-z]{2}))?$", re.I)
+
+    def __init__(self, tag, languages):
+        self.tag = tag
+        self.languages = languages
+        self.normalized_tag = self._normalize(tag)
+
+    def __eq__(self, other_tag):
+        return self.normalized_tag == self._normalize(other_tag)
+
+    def __lt__(self, other_tag):
+        return str(self) < self._normalize(other)
+
+    def __str__(self):
+        return self.normalized_tag
+
+    def __repr__(self):
+        return '<LanguageTag "{}">'.format(str(self))
+
+    def _normalize(self, tag):
+        if not tag:
+            raise ValueError('empty language tag')
+        languages = {language.lower().replace('-', '_'): language
+                     for language in self.languages}
+        try:
+            return languages[tag.lower().replace('-', '_')]
+        except KeyError:
+            try:
+                return languages[self._LANGUAGE_RE.match(tag).group(1).lower()]
+            except (KeyError, AttributeError):
+                raise ValueError('unsupported language: {!r}'.format(tag))

language_tool_python/match.py

@@ -0,0 +1,119 @@
+import unicodedata
+from collections import OrderedDict
+from functools import total_ordering
+
+def get_match_ordered_dict():
+    slots = OrderedDict([
+        ('ruleId', str), 
+        ('message', str),
+        ('replacements', list),
+        ('offsetInContext', int), 
+        ('context', str), 
+        ('offset', int), 
+        ('errorLength', int),
+        ('category', str), 
+        ('ruleIssueType', str),
+        ('sentence', str), 
+    ])
+    return slots
+
+""" Sample match JSON:
+    {
+        'message': 'Possible spelling mistake found.', 
+        'shortMessage': 'Spelling mistake', 
+        'replacements': [{'value': 'newt'}, {'value': 'not'}, {'value': 'new', 'shortDescription': 'having just been made'}, {'value': 'news'}, {'value': 'foot', 'shortDescription': 'singular'}, {'value': 'root', 'shortDescription': 'underground organ of a plant'}, {'value': 'boot'}, {'value': 'noon'}, {'value': 'loot', 'shortDescription': 'plunder'}, {'value': 'moot'}, {'value': 'Root'}, {'value': 'soot', 'shortDescription': 'carbon black'}, {'value': 'newts'}, {'value': 'nook'}, {'value': 'Lieut'}, {'value': 'coot'}, {'value': 'hoot'}, {'value': 'toot'}, {'value': 'snoot'}, {'value': 'neut'}, {'value': 'nowt'}, {'value': 'Noor'}, {'value': 'noob'}], 
+        'offset': 8, 
+        'length': 4, 
+        'context': {'text': 'This is noot okay. ', 'offset': 8, 'length': 4}, 'sentence': 'This is noot okay.', 
+        'type': {'typeName': 'Other'}, 
+        'rule': {'id': 'MORFOLOGIK_RULE_EN_US', 'description': 'Possible spelling mistake', 'issueType': 'misspelling', 'category': {'id': 'TYPOS', 'name': 'Possible Typo'}}, 
+        'ignoreForIncompleteSentence': False, 
+        'contextForSureMatch': 0
+    }
+
+"""
+def auto_type(obj):
+    try:
+        return int(obj)
+    except ValueError:
+        try:
+            return float(obj)
+        except ValueError:
+            return obj
+
+@total_ordering
+class Match:
+    """Hold information about where a rule matches text."""
+    def __init__(self, attrib):
+        # Process rule.
+        attrib['category'] = attrib['rule']['category']['id']
+        attrib['ruleId'] = attrib['rule']['id']
+        attrib['ruleIssueType'] = attrib['rule']['issueType']
+        del attrib['rule']
+        # Process context.
+        attrib['offsetInContext'] = attrib['context']['offset']
+        attrib['context'] = attrib['context']['text']
+        # Process replacements.
+        attrib['replacements'] = [r['value'] for r in attrib['replacements']]
+        # Rename error length.
+        attrib['errorLength'] = attrib['length']
+        # Normalize unicode
+        attrib['message'] = unicodedata.normalize("NFKC", attrib['message'])
+        # Store objects on self.
+        for k, v in attrib.items():
+            setattr(self, k, v)
+
+    def __repr__(self):
+        def _ordered_dict_repr():
+            slots = list(get_match_ordered_dict())
+            slots += list(set(self.__dict__).difference(slots))
+            attrs = [slot for slot in slots
+                     if slot in self.__dict__ and not slot.startswith('_')]
+            return '{{{}}}'.format(
+                ', '.join([
+                    '{!r}: {!r}'.format(attr, getattr(self, attr))
+                    for attr in attrs
+                ])
+            )
+
+        return '{}({})'.format(self.__class__.__name__, _ordered_dict_repr())
+
+    def __str__(self):
+        ruleId = self.ruleId
+        s = 'Offset {}, length {}, Rule ID: {}'.format(
+            self.offset, self.errorLength, ruleId)
+        if self.message:
+            s += '\nMessage: {}'.format(self.message)
+        if self.replacements:
+            s += '\nSuggestion: {}'.format('; '.join(self.replacements))
+        s += '\n{}\n{}'.format(
+            self.context, ' ' * self.offsetInContext + '^' * self.errorLength
+        )
+        return s
+
+    @property
+    def matchedText(self):
+        """ Returns the text that garnered the error (without its surrounding context).
+        """
+        return self.context[self.offsetInContext:self.offsetInContext+self.errorLength]
+
+    def __eq__(self, other):
+        return list(self) == list(other)
+
+    def __lt__(self, other):
+        return list(self) < list(other)
+
+    def __iter__(self):
+        return iter(getattr(self, attr) for attr in get_match_ordered_dict())
+
+    def __setattr__(self, key, value):
+        try:
+            value = get_match_ordered_dict()[key](value)
+        except KeyError:
+            return
+        super().__setattr__(key, value)
+
+    def __getattr__(self, name):
+        if name not in get_match_ordered_dict():
+            raise AttributeError('{!r} object has no attribute {!r}'
+                                 .format(self.__class__.__name__, name))

language_tool_python/server.py

@@ -0,0 +1,399 @@
+from typing import Dict, List
+
+import atexit
+import http.client
+import json
+import os
+import re
+import requests
+import socket
+import subprocess
+import threading
+import urllib.parse
+
+from .config_file import LanguageToolConfig
+from .download_lt import download_lt, LTP_DOWNLOAD_VERSION
+from .language_tag import LanguageTag
+from .match import Match
+from .utils import (
+    correct,
+    parse_url, get_locale_language,
+    get_language_tool_directory, get_server_cmd,
+    FAILSAFE_LANGUAGE, startupinfo,
+    LanguageToolError, ServerError, PathError
+)
+
+
+DEBUG_MODE = False
+
+# Keep track of running server PIDs in a global list. This way,
+# we can ensure they're killed on exit.
+RUNNING_SERVER_PROCESSES: List[subprocess.Popen] = []
+
+
+class LanguageTool:
+    """Main class used for checking text against different rules.
+    LanguageTool v2 API documentation:
+    https://languagetool.org/http-api/swagger-ui/#!/default/post_check
+    """
+    _MIN_PORT = 8081
+    _MAX_PORT = 8999
+    _TIMEOUT = 5 * 60
+    _remote = False
+    _port = _MIN_PORT
+    _server: subprocess.Popen = None
+    _consumer_thread: threading.Thread = None
+    _PORT_RE = re.compile(r"(?:https?://.*:|port\s+)(\d+)", re.I)
+
+    def __init__(
+            self, language=None, motherTongue=None,
+            remote_server=None, newSpellings=None,
+            new_spellings_persist=True,
+            host=None, config=None,
+            language_tool_download_version: str = LTP_DOWNLOAD_VERSION
+    ):
+        self.language_tool_download_version = language_tool_download_version
+        self._new_spellings = None
+        self._new_spellings_persist = new_spellings_persist
+        self._host = host or socket.gethostbyname('localhost')
+
+        if remote_server:
+            assert config is None, "cannot pass config file to remote server"
+        self.config = LanguageToolConfig(config) if config else None
+
+        if remote_server is not None:
+            self._remote = True
+            self._url = parse_url(remote_server)
+            self._url = urllib.parse.urljoin(self._url, 'v2/')
+            self._update_remote_server_config(self._url)
+        elif not self._server_is_alive():
+            self._start_server_on_free_port()
+        if language is None:
+            try:
+                language = get_locale_language()
+            except ValueError:
+                language = FAILSAFE_LANGUAGE
+        if newSpellings:
+            self._new_spellings = newSpellings
+            self._register_spellings(self._new_spellings)
+        self._language = LanguageTag(language, self._get_languages())
+        self.motherTongue = motherTongue
+        self.disabled_rules = set()
+        self.enabled_rules = set()
+        self.disabled_categories = set()
+        self.enabled_categories = set()
+        self.enabled_rules_only = False
+        self.preferred_variants = set()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def __del__(self):
+        self.close()
+
+    def __repr__(self):
+        return '{}(language={!r}, motherTongue={!r})'.format(
+            self.__class__.__name__, self.language, self.motherTongue)
+
+    def close(self):
+        if self._server_is_alive():
+            self._terminate_server()
+        if not self._new_spellings_persist and self._new_spellings:
+            self._unregister_spellings()
+            self._new_spellings = []
+
+    @property
+    def language(self):
+        """The language to be used."""
+        return self._language
+
+    @language.setter
+    def language(self, language):
+        self._language = LanguageTag(language, self._get_languages())
+        self.disabled_rules.clear()
+        self.enabled_rules.clear()
+
+    @property
+    def motherTongue(self):
+        """The user's mother tongue or None.
+        The mother tongue may also be used as a source language for
+        checking bilingual texts.
+        """
+        return self._motherTongue
+
+    @motherTongue.setter
+    def motherTongue(self, motherTongue):
+        self._motherTongue = (
+            None if motherTongue is None
+            else LanguageTag(motherTongue, self._get_languages())
+        )
+
+    @property
+    def _spell_checking_categories(self):
+        return {'TYPOS'}
+
+    def check(self, text: str) -> List[Match]:
+        """Match text against enabled rules."""
+        url = urllib.parse.urljoin(self._url, 'check')
+        response = self._query_server(url, self._create_params(text))
+        matches = response['matches']
+        return [Match(match) for match in matches]
+
+    def _create_params(self, text: str) -> Dict[str, str]:
+        params = {'language': str(self.language), 'text': text}
+        if self.motherTongue is not None:
+            params['motherTongue'] = self.motherTongue
+        if self.disabled_rules:
+            params['disabledRules'] = ','.join(self.disabled_rules)
+        if self.enabled_rules:
+            params['enabledRules'] = ','.join(self.enabled_rules)
+        if self.enabled_rules_only:
+            params['enabledOnly'] = 'true'
+        if self.disabled_categories:
+            params['disabledCategories'] = ','.join(self.disabled_categories)
+        if self.enabled_categories:
+            params['enabledCategories'] = ','.join(self.enabled_categories)
+        if self.preferred_variants:
+            params['preferredVariants'] = ','.join(self.preferred_variants)
+        return params
+
+    def correct(self, text: str) -> str:
+        """Automatically apply suggestions to the text."""
+        return correct(text, self.check(text))
+
+    def enable_spellchecking(self):
+        """Enable spell-checking rules."""
+        self.disabled_categories.difference_update(
+            self._spell_checking_categories
+        )
+
+    def disable_spellchecking(self):
+        """Disable spell-checking rules."""
+        self.disabled_categories.update(self._spell_checking_categories)
+
+    @staticmethod
+    def _get_valid_spelling_file_path() -> str:
+        library_path = get_language_tool_directory()
+        spelling_file_path = os.path.join(
+            library_path, "org/languagetool/resource/en/hunspell/spelling.txt"
+        )
+        if not os.path.exists(spelling_file_path):
+            raise FileNotFoundError(
+                "Failed to find the spellings file at {}\n "
+                "Please file an issue at "
+                "https://github.com/jxmorris12/language_tool_python/issues"
+                .format(spelling_file_path))
+        return spelling_file_path
+
+    def _register_spellings(self, spellings):
+        spelling_file_path = self._get_valid_spelling_file_path()
+        with (
+            open(spelling_file_path, "a+", encoding='utf-8')
+        ) as spellings_file:
+            spellings_file.write(
+                "\n" + "\n".join([word for word in spellings])
+            )
+        if DEBUG_MODE:
+            print("Registered new spellings at {}".format(spelling_file_path))
+
+    def _unregister_spellings(self):
+        spelling_file_path = self._get_valid_spelling_file_path()
+        with (
+            open(spelling_file_path, 'r+', encoding='utf-8')
+        ) as spellings_file:
+            spellings_file.seek(0, os.SEEK_END)
+            for _ in range(len(self._new_spellings)):
+                while spellings_file.read(1) != '\n':
+                    spellings_file.seek(spellings_file.tell() - 2, os.SEEK_SET)
+                spellings_file.seek(spellings_file.tell() - 2, os.SEEK_SET)
+            spellings_file.seek(spellings_file.tell() + 1, os.SEEK_SET)
+            spellings_file.truncate()
+        if DEBUG_MODE:
+            print(
+                "Unregistered new spellings at {}".format(spelling_file_path)
+            )
+
+    def _get_languages(self) -> set:
+        """Get supported languages (by querying the server)."""
+        self._start_server_if_needed()
+        url = urllib.parse.urljoin(self._url, 'languages')
+        languages = set()
+        for e in self._query_server(url, num_tries=1):
+            languages.add(e.get('code'))
+            languages.add(e.get('longCode'))
+        languages.add("auto")
+        return languages
+
+    def _start_server_if_needed(self):
+        # Start server.
+        if not self._server_is_alive() and self._remote is False:
+            self._start_server_on_free_port()
+
+    def _update_remote_server_config(self, url):
+        self._url = url
+        self._remote = True
+
+    def _query_server(self, url, params=None, num_tries=2):
+        if DEBUG_MODE:
+            print('_query_server url:', url, 'params:', params)
+        for n in range(num_tries):
+            try:
+                with (
+                    requests.get(url, params=params, timeout=self._TIMEOUT)
+                ) as response:
+                    try:
+                        return response.json()
+                    except json.decoder.JSONDecodeError as e:
+                        if DEBUG_MODE:
+                            print(
+                                'URL {} and params {} '
+                                'returned invalid JSON response: {}'
+                                .format(url, params, e)
+                            )
+                            print(response)
+                            print(response.content)
+                        raise LanguageToolError(response.content.decode())
+            except (IOError, http.client.HTTPException) as e:
+                if self._remote is False:
+                    self._terminate_server()
+                    self._start_local_server()
+                if n + 1 >= num_tries:
+                    raise LanguageToolError('{}: {}'.format(self._url, e))
+
+    def _start_server_on_free_port(self):
+        while True:
+            self._url = 'http://{}:{}/v2/'.format(self._host, self._port)
+            try:
+                self._start_local_server()
+                break
+            except ServerError:
+                if self._MIN_PORT <= self._port < self._MAX_PORT:
+                    self._port += 1
+                else:
+                    raise
+
+    def _start_local_server(self):
+        # Before starting local server, download language tool if needed.
+        download_lt(self.language_tool_download_version)
+        err = None
+        try:
+            if DEBUG_MODE:
+                if self._port:
+                    print(
+                        'language_tool_python initializing with port:',
+                        self._port
+                    )
+                if self.config:
+                    print(
+                        'language_tool_python initializing '
+                        'with temporary config file:',
+                        self.config.path
+                    )
+            server_cmd = get_server_cmd(self._port, self.config)
+        except PathError as e:
+            # Can't find path to LanguageTool.
+            err = e
+        else:
+            # Need to PIPE all handles: http://bugs.python.org/issue3905
+            self._server = subprocess.Popen(
+                server_cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+                startupinfo=startupinfo
+            )
+            global RUNNING_SERVER_PROCESSES
+            RUNNING_SERVER_PROCESSES.append(self._server)
+
+            match = None
+            while True:
+                line = self._server.stdout.readline()
+                if not line:
+                    break
+                match = self._PORT_RE.search(line)
+                if match:
+                    port = int(match.group(1))
+                    if port != self._port:
+                        raise LanguageToolError(
+                            'requested port {}, but got {}'
+                            .format(self._port, port)
+                        )
+                    break
+            if not match:
+                err_msg = self._terminate_server()
+                match = self._PORT_RE.search(err_msg)
+                if not match:
+                    raise LanguageToolError(err_msg)
+                port = int(match.group(1))
+                if port != self._port:
+                    raise LanguageToolError(err_msg)
+
+        if self._server:
+            self._consumer_thread = threading.Thread(
+                target=lambda: _consume(self._server.stdout))
+            self._consumer_thread.daemon = True
+            self._consumer_thread.start()
+        else:
+            # Couldn't start the server, so maybe there is already one running.
+            if err:
+                raise Exception(err)
+            else:
+                raise ServerError(
+                    'Server running; don\'t start a server here.'
+                )
+
+    def _server_is_alive(self):
+        return self._server and self._server.poll() is None
+
+    def _terminate_server(self):
+        LanguageToolError_message = ''
+        try:
+            self._server.terminate()
+        except OSError:
+            pass
+        try:
+            LanguageToolError_message = self._server.communicate()[1].strip()
+        except (IOError, ValueError):
+            pass
+        try:
+            self._server.stdout.close()
+        except IOError:
+            pass
+        try:
+            self._server.stdin.close()
+        except IOError:
+            pass
+        try:
+            self._server.stderr.close()
+        except IOError:
+            pass
+        self._server = None
+        return LanguageToolError_message
+
+
+class LanguageToolPublicAPI(LanguageTool):
+    """Language tool client of the official API."""
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            *args, remote_server='https://languagetool.org/api/', **kwargs
+        )
+
+
+@atexit.register
+def terminate_server():
+    """Terminate the server."""
+    for proc in RUNNING_SERVER_PROCESSES:
+        proc.terminate()
+
+
+def _consume(stdout):
+    """Consume/ignore the rest of the server output.
+    Without this, the server will end up hanging due to the buffer
+    filling up.
+    """
+    while stdout.readline():
+        pass

language_tool_python/utils.py

@@ -0,0 +1,179 @@
+from typing import List, Tuple
+
+import glob
+import locale
+import os
+import subprocess
+import urllib.parse
+import urllib.request
+
+from .config_file import LanguageToolConfig
+from .match import Match
+from .which import which
+
+JAR_NAMES = [
+    'languagetool-server.jar',
+    'languagetool-standalone*.jar',  # 2.1
+    'LanguageTool.jar',
+    'LanguageTool.uno.jar'
+]
+FAILSAFE_LANGUAGE = 'en'
+
+LTP_PATH_ENV_VAR = "LTP_PATH"  # LanguageTool download path
+
+# Directory containing the LanguageTool jar file:
+LTP_JAR_DIR_PATH_ENV_VAR = "LTP_JAR_DIR_PATH"
+
+# https://mail.python.org/pipermail/python-dev/2011-July/112551.html
+
+if os.name == 'nt':
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+else:
+    startupinfo = None
+
+
+class LanguageToolError(Exception):
+    pass
+
+
+class ServerError(LanguageToolError):
+    pass
+
+
+class JavaError(LanguageToolError):
+    pass
+
+
+class PathError(LanguageToolError):
+    pass
+
+
+def parse_url(url_str):
+    """ Parses a URL string, and adds 'http' if necessary. """
+    if 'http' not in url_str:
+        url_str = 'http://' + url_str
+
+    return urllib.parse.urlparse(url_str).geturl()
+
+
+def _4_bytes_encoded_positions(text: str) -> List[int]:
+    """Return a list of positions of 4-byte encoded characters in the text."""
+    positions = []
+    char_index = 0
+    for char in text:
+        if len(char.encode('utf-8')) == 4:
+            positions.append(char_index)
+            # Adding 1 to the index because 4 byte characters are
+            # 2 bytes in length in LanguageTool, instead of 1 byte in Python.
+            char_index += 1
+        char_index += 1
+    return positions
+
+
+def correct(text: str, matches: List[Match]) -> str:
+    """Automatically apply suggestions to the text."""
+    # Get the positions of 4-byte encoded characters in the text because without 
+    # carrying out this step, the offsets of the matches could be incorrect.
+    for match in matches:
+        match.offset -= sum(1 for i in _4_bytes_encoded_positions(text) if i <= match.offset)
+    ltext = list(text)
+    matches = [match for match in matches if match.replacements]
+    errors = [ltext[match.offset:match.offset + match.errorLength]
+              for match in matches]
+    correct_offset = 0
+    for n, match in enumerate(matches):
+        frompos, topos = (correct_offset + match.offset,
+                          correct_offset + match.offset + match.errorLength)
+        if ltext[frompos:topos] != errors[n]:
+            continue
+        repl = match.replacements[0]
+        ltext[frompos:topos] = list(repl)
+        correct_offset += len(repl) - len(errors[n])
+    return ''.join(ltext)
+
+
+def get_language_tool_download_path() -> str:
+    # Get download path from environment or use default.
+    download_path = os.environ.get(
+        LTP_PATH_ENV_VAR,
+        os.path.join(os.path.expanduser("~"), ".cache", "language_tool_python")
+    )
+    return download_path
+
+
+def find_existing_language_tool_downloads(download_folder: str) -> List[str]:
+    language_tool_path_list = [
+        path for path in
+        glob.glob(os.path.join(download_folder, 'LanguageTool*'))
+        if os.path.isdir(path)
+    ]
+    return language_tool_path_list
+
+
+def get_language_tool_directory() -> str:
+    """Get LanguageTool directory."""
+    download_folder = get_language_tool_download_path()
+    if not os.path.isdir(download_folder):
+        raise NotADirectoryError(
+            "LanguageTool directory path is not a valid directory {}."
+            .format(download_folder)
+        )
+    language_tool_path_list = find_existing_language_tool_downloads(
+        download_folder
+    )
+
+    if not len(language_tool_path_list):
+        raise FileNotFoundError(
+            'LanguageTool not found in {}.'.format(download_folder)
+        )
+
+    # Return the latest version found in the directory.
+    return max(language_tool_path_list)
+
+
+def get_server_cmd(
+        port: int = None, config: LanguageToolConfig = None
+) -> List[str]:
+    java_path, jar_path = get_jar_info()
+    cmd = [java_path, '-cp', jar_path,
+           'org.languagetool.server.HTTPServer']
+
+    if port is not None:
+        cmd += ['-p', str(port)]
+
+    if config is not None:
+        cmd += ['--config', config.path]
+
+    return cmd
+
+
+def get_jar_info() -> Tuple[str, str]:
+    java_path = which('java')
+    if not java_path:
+        raise JavaError("can't find Java")
+
+    # Use the env var to the jar directory if it is defined
+    # otherwise look in the download directory
+    jar_dir_name = os.environ.get(
+        LTP_JAR_DIR_PATH_ENV_VAR,
+        get_language_tool_directory()
+    )
+    jar_path = None
+    for jar_name in JAR_NAMES:
+        for jar_path in glob.glob(os.path.join(jar_dir_name, jar_name)):
+            if os.path.isfile(jar_path):
+                break
+        else:
+            jar_path = None
+        if jar_path:
+            break
+    else:
+        raise PathError("can't find languagetool-standalone in {!r}"
+                        .format(jar_dir_name))
+    return java_path, jar_path
+
+
+def get_locale_language():
+    """Get the language code for the current locale setting."""
+    return locale.getlocale()[0] or locale.getdefaultlocale()[0]

language_tool_python/which.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+"""Cross-platform which command."""
+
+import os
+import sys
+
+
+__all__ = ['which']
+
+WIN_ALLOW_CROSS_ARCH = True
+
+def which(program):
+    """Identify the location of an executable file."""
+    if os.path.split(program)[0]:
+        program_path = find_exe(program)
+        if program_path:
+            return program_path
+    else:
+        for path in get_path_list():
+            program_path = find_exe(os.path.join(path, program))
+            if program_path:
+                return program_path
+    return None
+
+
+def is_exe(path):
+    return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def _get_path_list():
+    return os.environ['PATH'].split(os.pathsep)
+
+
+if os.name == 'nt':
+    def find_exe(program):
+        root, ext = os.path.splitext(program)
+        if ext:
+            if is_exe(program):
+                return program
+        else:
+            for ext in os.environ['PATHEXT'].split(os.pathsep):
+                program_path = root + ext.lower()
+                if is_exe(program_path):
+                    return program_path
+        return None
+
+    def get_path_list():
+        paths = _get_path_list()
+        if WIN_ALLOW_CROSS_ARCH:
+            alt_sys_path = os.path.expandvars(r"$WINDIR\Sysnative")
+            if os.path.isdir(alt_sys_path):
+                paths.insert(0, alt_sys_path)
+            else:
+                alt_sys_path = os.path.expandvars(r"$WINDIR\SysWOW64")
+                if os.path.isdir(alt_sys_path):
+                    paths.append(alt_sys_path)
+        return paths
+
+else:
+    def find_exe(program):
+        return program if is_exe(program) else None
+
+    get_path_list = _get_path_list
+
+
+def main():
+    for arg in sys.argv[1:]:
+        path = which(arg)
+        if path:
+            print(path)
+
+
+if __name__ == '__main__':
+    sys.exit(main())