LXTAGGER_WS_API_KEY = 'access_key_goes_here'
LXTAGGER_WS_API_URL = 'https://portulanclarin.net/workbench/lx-tagger/api/'

try:
    import requests
except:
    !pip3 install requests
    import requests
try:
    import matplotlib.pyplot as plt
except:
    !pip3 install matplotlib
    import matplotlib.pyplot as plt
import collections

class WSException(Exception):
    'Webservice Exception'
    def __init__(self, errordata):
        "errordata is a dict returned by the webservice with details about the error"
        super().__init__(self)
        assert isinstance(errordata, dict)
        self.message = errordata["message"]
        # see https://json-rpc.readthedocs.io/en/latest/exceptions.html for more info
        # about JSON-RPC error codes
        if -32099 <= errordata["code"] <= -32000:  # Server Error
            if errordata["data"]["type"] == "WebServiceException":
                self.message += f": {errordata['data']['message']}"
            else:
                self.message += f": {errordata['data']!r}"
    def __str__(self):
        return self.message

def tag(text, format):
    '''
    Arguments
        text: a string with a maximum of 4000 characters, Portuguese text, with
             the input to be processed
        format: either 'CINTIL', 'CONLL' or 'JSON'

    Returns a string with the output according to specification in
       https://portulanclarin.net/workbench/lx-tagger/
    
    Raises a WSException if an error occurs.
    '''

    request_data = {
        'method': 'tag',
        'jsonrpc': '2.0',
        'id': 0,
        'params': {
            'text': text,
            'format': format,
            'key': LXTAGGER_WS_API_KEY,
        },
    }
    request = requests.post(LXTAGGER_WS_API_URL, json=request_data)
    response_data = request.json()
    if "error" in response_data:
        raise WSException(response_data["error"])
    else:
        return response_data["result"]

text = '''Esta frase serve para testar o funcionamento do tagger. Esta outra
frase faz o mesmo.'''
# the CONLL annotation format is a popular format for annotating part of speech
result = tag(text, format="CONLL")
print(result)

annotated_text = tag(text, format="JSON")
for pnum, paragraph in enumerate(annotated_text, start=1): # enumerate paragraphs in text, starting at 1
    print(f"paragraph {pnum}:")
    for snum, sentence in enumerate(paragraph, start=1): # enumerate sentences in paragraph, starting at 1
        print(f"  sentence {snum}:")
        for tnum, token in enumerate(sentence, start=1): # enumerate tokens in sentence, starting at 1
            print(f"    token {tnum}: {token!r}")  # print a token representation

# A plain text version of this book is available from our Gitbub repository:
sample_text_url = "https://github.com/portulanclarin/jupyter-notebooks/raw/main/sample-data/viagensnaminhaterra.txt"

req = requests.get(sample_text_url)
sample_text_lines = req.text.splitlines()

num_lines = len(sample_text_lines)
print(f"The downloaded text contains {num_lines} lines")

# discard whitespace at beginning and end of each line:
sample_text_lines = [line.strip() for line in sample_text_lines]

# discard empty lines
sample_text_lines = [line for line in sample_text_lines if line]

# how many lines do we have left?
num_lines = len(sample_text_lines)
print(f"After discarding empty lines we are left with {num_lines} non-empty lines")

def slice_into_chunks(lines, max_chunk_size=4000):
    chunk, chunk_size = [], 0
    for lnum, line in enumerate(lines, start=1):
        if (chunk_size + len(line)) <= max_chunk_size:
            chunk.append(line)
            chunk_size += len(line) + 1
            # the + 1 above is for the newline character terminating each line
        else:
            yield "\n".join(chunk)
            if len(line) > max_chunk_size:
                print(f"line {lnum} is longer than 4000 characters; truncating")
                line = line[:4000]
            chunk, chunk_size = [line], len(line) + 1
    if chunk:
        yield "\n".join(chunk)

chunks = list(slice_into_chunks(sample_text_lines))
annotated_text = [] # annotated paragraphs will be stored here
chunks_processed = 0  # this variable keeps track of which chunks have been processed already
print(f"There are {len(chunks)} chunks to be annotated")

for cnum, chunk in enumerate(chunks[chunks_processed:], start=chunks_processed+1):
    try:
        annotated_text.extend(tag(chunk, format="JSON"))
        chunks_processed = cnum
        # print one dot for each annotated chunk to get some progress feedback
        print(".", end="", flush=True)
    except Exception as exc:
        chunk_preview = chunk[:100] + "[...]" if len(chunk) > 100 else chunk
        print(
            f"\nError: annotation of chunk {cnum} failed ({exc}); chunk contents:\n\n{chunk_preview}\n\n"
        )
        break

%matplotlib inline

tag_frequencies = collections.Counter(
        token["pos"]
        for paragraph in annotated_text
        for sentence in paragraph
        for token in sentence
).most_common()

tags = [tag for tag, _ in tag_frequencies[:9]]
freqs = [freq for _, freq in tag_frequencies[:9]]

tags.append("other")
freqs.append(sum(freq for _, freq in tag_frequencies[10:]))

plt.rcParams['figure.figsize'] = [10, 10]
fig1, ax1 = plt.subplots()
ax1.pie(freqs, labels=tags, autopct='%1.1f%%', startangle=90)
ax1.axis('equal')  # equal aspect ratio ensures that pie is drawn as a circle.

plt.show()
# To learn more about matplotlib visit https://matplotlib.org/

def get_key_status():
    '''Returns a string with the detailed status of the webservice access key'''
    
    request_data = {
        'method': 'key_status',
        'jsonrpc': '2.0',
        'id': 0,
        'params': {
            'key': LXTAGGER_WS_API_KEY,
        },
    }
    request = requests.post(LXTAGGER_WS_API_URL, json=request_data)
    response_data = request.json()
    if "error" in response_data:
        raise WSException(response_data["error"])
    else:
        return response_data["result"]

get_key_status()