Finally bump pydantic version

README.md

@@ -28,22 +28,3 @@ python3 setup.py install
 
 ## Reference
 A comment provided for each function.
-
-## Termforces
-Termforces is a CLI wrapper for scraper. Currently it has a python module and a shell script
-
-## Quickstart
-Use `termforces login <username>` to login, it will store your session,
-so you're not needed to do it often.
-
-Then enter a directory you want and use `termforces strap --contest-id <contest_id> --indices <indices>`.
-
-Indices should be separated with space, i.e `termforces strap --contest-id 1329 --indices "A B1 B2 C D E"`.
-It will create folders `problem$index` for each index you specified.
-
-You may also specify template folder with `--template <path-to-template-folder>`, in this case script will copy
-its contents to all problem subfolders.
-
-You can check your results with `termforces results` and submit files with `termforces submit <filename>`.
-Script will determine contest id from parent .rc file and problem index from folder name. You may run it either from parent or from child
-directory.

codeforces_scraper/models.py

@@ -71,8 +71,8 @@ class APIModel(BaseModel):
 
 class JudgeProtocol(APIModel):
     manual: bool
-    protocol: Optional[str]
+    protocol: Optional[str] = None
-    verdict: Optional[str]
+    verdict: Optional[str] = None
 
 
 class BlogEntry(APIModel):
@@ -81,7 +81,7 @@ class BlogEntry(APIModel):
     creation_time_seconds: int
     author_handle: str
     title: str
-    content: Optional[str]
+    content: Optional[str] = None
     locale: str
     modification_time_seconds: int
     allow_view_history: bool
@@ -95,7 +95,7 @@ class Comment(APIModel):
     commentator_handle: str
     locale: str
     text: str
-    parent_comment_id: Optional[int]
+    parent_comment_id: Optional[int] = None
     rating: int
 
 
@@ -126,26 +126,26 @@ class Member(APIModel):
 
 
 class Problem(APIModel):
-    contest_id: Optional[int]
+    contest_id: Optional[int] = None
-    problem_set_name: Optional[str]
+    problem_set_name: Optional[str] = None
     index: str
     name: str
     type: str
-    points: Optional[float]
+    points: Optional[float] = None
-    rating: Optional[int]
+    rating: Optional[int] = None
     tags: List[str]
 
 
 class User(APIModel):
     handle: str
-    email: Optional[str]
+    email: Optional[str] = None
-    vk_id: Optional[str]
+    vk_id: Optional[str] = None
-    open_id: Optional[str]
+    open_id: Optional[str] = None
-    first_name: Optional[str]
+    first_name: Optional[str] = None
-    last_name: Optional[str]
+    last_name: Optional[str] = None
-    country: Optional[str]
+    country: Optional[str] = None
-    city: Optional[str]
+    city: Optional[str] = None
-    organization: Optional[str]
+    organization: Optional[str] = None
     contribution: int
     rank: str
     rating: int
@@ -162,11 +162,11 @@ class Party(APIModel):
     contest_id: int
     members: List[Member]
     participant_type: str
-    team_id: Optional[int]
+    team_id: Optional[int] = None
-    team_name: Optional[str]
+    team_name: Optional[str] = None
     ghost: bool
-    room: Optional[int]
+    room: Optional[int] = None
-    start_time_seconds: Optional[int]
+    start_time_seconds: Optional[int] = None
 
 
 class Submission(APIModel):
@@ -177,12 +177,12 @@ class Submission(APIModel):
     problem: Problem
     author: Party
     programming_language: str
-    verdict: Optional[Verdict]
+    verdict: Optional[Verdict] = None
     testset: str
     passed_test_count: int
     time_consumed_millis: int
     memory_consumed_bytes: int
-    points: Optional[float]
+    points: Optional[float] = None
 
 
 class Contest(APIModel):
@@ -192,17 +192,17 @@ class Contest(APIModel):
     phase: ContestPhase
     frozen: bool
     duration_seconds: bool
-    start_time_seconds: Optional[int]
+    start_time_seconds: Optional[int] = None
-    relative_time_seconds: Optional[int]
+    relative_time_seconds: Optional[int] = None
-    prepared_by: Optional[str]
+    prepared_by: Optional[str] = None
-    website_url: Optional[str]
+    website_url: Optional[str] = None
-    description: Optional[str]
+    description: Optional[str] = None
-    difficulty: Optional[int]
+    difficulty: Optional[int] = None
-    kind: Optional[str]
+    kind: Optional[str] = None
-    icpc_region: Optional[str]
+    icpc_region: Optional[str] = None
-    country: Optional[str]
+    country: Optional[str] = None
-    city: Optional[str]
+    city: Optional[str] = None
-    season: Optional[str]
+    season: Optional[str] = None
 
 
 class Hack(APIModel):
@@ -211,8 +211,8 @@ class Hack(APIModel):
     hacker: Party
     defender: Party
     problem: Problem
-    test: Optional[str]
+    test: Optional[str] = None
-    judge_protocol = JudgeProtocol
+    judge_protocol: JudgeProtocol
 
 
 class ProblemResult(APIModel):
@@ -232,3 +232,8 @@ class RanklistRow(APIModel):
     unsuccessful_hack_count: int
     problem_result: List[ProblemResult]
     last_submission_time_seconds: int
+
+
+class Sample(BaseModel):
+    s_in: str
+    s_out: str

codeforces_scraper/scraper.py

@@ -3,8 +3,8 @@ import requests
 from requests import Session
 from bs4 import BeautifulSoup as bs
 
-from codeforces_scraper.utils import get_token, get_messages, create_jar
+from codeforces_scraper.utils import get_token, get_messages, create_jar, unfuck_multitest_sample
-from codeforces_scraper.models import Submission, Problem
+from codeforces_scraper.models import Submission, Problem, Sample
 from typing import List
 
 
@@ -114,8 +114,10 @@ class Scraper:
             raise ScraperError('Submitting while not logged in')
         url = f'contest/{contest_id}/submit'
         submit_page_response = self.get(url)
-        for message in get_messages(submit_page_response):
+        # FIXME: Now some pornography is in the messages, which is not displayed and
-            raise MessagedScrapError(message)
+        # is not an error
+        # for message in get_messages(submit_page_response):
+        #     raise MessagedScrapError(message)
         token = get_token(submit_page_response)
         payload = {
             'csrf_token': token,
@@ -167,7 +169,7 @@ class Scraper:
             }
         else:
             params = {'contestId': contest_id}
-        return [Submission.parse_obj(x) for x in self.api_request('contest.status', params)]
+        return [Submission.model_validate(x) for x in self.api_request('contest.status', params)]
 
     def get_contest_tasks(self, contest_id: int) -> List[Problem]:
         """Get all tasks in contest with id ``contest_id``"""
@@ -177,6 +179,16 @@ class Scraper:
         }
         return self.api_request('contest.standings', params)['problems']
 
+    def get_samples(self, contest_id: int, problem_index: str) -> List[Sample]:
+        url = f'contest/{contest_id}/problem/{problem_index}'
+        page_response = self.get(url)
+        soup = bs(page_response.text, 'lxml')
+        samples = soup.find(attrs={'class': 'sample-tests'}).find(attrs={'class': 'sample-test'})
+        inputs = [unfuck_multitest_sample(str(div_input.find(name='pre')))
+                  for div_input in samples.find_all(attrs={'class': 'input'})]
+        outputs = [div_output.find(name='pre').get_text() for div_output in samples.find_all(attrs={'class', 'output'})]
+        return [Sample(s_in=s_in, s_out=s_out) for (s_in, s_out) in zip(inputs, outputs)]
+
     def get(self, sub_url='', **kwargs):
         """Make a GET request to BASE_URL"""
         url = self.base_url + '/' + sub_url

codeforces_scraper/utils.py

@@ -8,6 +8,15 @@ MESSAGE_GREP_STRING = r'Codeforces\.showMessage\('
 # TODO: Grep for Codeforces.showMessage(" to find message, that has been sent
 
 
+def unfuck_multitest_sample(sample_input: str) -> str:
+    div_class_regex = '<div class="[a-zA-Z0-9- ]*">'
+    sample_input = re.sub(div_class_regex, '', sample_input)
+    sample_input = re.sub('</div>', '\n', sample_input)
+    sample_input = re.sub('<pre>', '', sample_input)
+    sample_input = re.sub('</pre>', '', sample_input)
+    return sample_input
+
+
 def create_jar(str_cookie: str):
     cookies = str_cookie.split(';')
     d = {}
@@ -27,6 +36,7 @@ def get_token(response: Response) -> str:
     return token
 
 
+# FIXME: More robust way to find messages
 def get_messages(response: Response) -> List[str]:
     text = response.text
     return re.findall(fr'{MESSAGE_GREP_STRING}\"(.+?)\"', text)

scripts/termforces

@@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-# Minimalistic script to interact with Codeforces
-# Written by thematdev in 2023
-
-
-load_rc() {
-    if [[ -e $1 ]]; then source $1; fi;
-}
-
-# Load .rc files
-load_rc ~/.config/termforces/termforces.rc
-load_rc ../termforces.rc
-load_rc termforces.rc
-
-login() {
-    if [[ $TF_PASS_CMD ]]
-    then
-        python3 -m termforces login "$1" --session-file "~/.config/termforces/termforces_cookies.json" --no-getpass <<< "$($TF_PASS_CMD)"
-    else
-        python3 -m termforces login "$1" --session-file "~/.config/termforces/termforces_cookies.json"
-    fi
-}
-
-whoami() {
-    python3 -m termforces whoami
-}
-
-submit() {
-    source_file=$1
-    # Calling from either parent or child directory
-    if [ $(dirname $1) = "." ]; then
-        p=$(pwd)
-    else
-        p=$(dirname $1)
-    fi
-    problem_index=${p:0-1}
-    python3 -m termforces submit --contest-id $contest_id $problem_index $source_file
-}
-
-results() {
-    python3 -m termforces results-my --contest-id $contest_id
-}
-
-strap() {
-    while [[ $# -gt 0 ]]
-    do
-        key="$1"
-        case $key in
-            -c|--contest-id)
-                contest_id="$2"
-                shift
-                shift
-                ;;
-            -i|--indices)
-                indices="$2"
-                shift
-                shift
-                ;;
-            -t|--template)
-                template="$2"
-                shift
-                shift
-                ;;
-            *)
-                POSITIONAL+=("$1")
-                shift
-                ;;
-        esac
-    done
-    read -a indices_arr <<< $indices
-    if [[ ! -d $template ]]; then echo "Template directory not found or not specified, will create empty"; fi;
-    for index in "${indices_arr[@]}"
-    do
-        if [[ -d $template ]]
-        then
-            cp -r $template problem$index
-        else
-            mkdir problem$index
-        fi
-    done
-    echo "contest_id=$contest_id" >> termforces.rc
-}
-
-"$@"

setup.py

@@ -2,20 +2,16 @@ import setuptools
 
 setuptools.setup(
         name='codeforces-scraper',
-        version='0.2.0-r1',
+        version='0.4.0',
         author='thematdev',
         author_email='thematdev@thematdev.org',
         description='Utility to do actions on codeforces',
-        packages=['codeforces_scraper', 'codeforces_scraper.assets',
+        packages=['codeforces_scraper', 'codeforces_scraper.assets'],
-                  'termforces', 'termforces.cmds'],
-        scripts=['scripts/termforces'],
         install_requires=[
             'bs4',
             'lxml',
             'pydantic',
             'requests',
-            'click',
-            'click_shell'
         ],
         python_requires='>=3.8',
         zip_safe=True,

termforces/__main__.py

@@ -1,6 +0,0 @@
-from termforces.termforces_shell import termforces_shell
-from termforces.cmds import *
-
-
-if __name__ == '__main__':
-    termforces_shell()

termforces/cmds/__init__.py

@@ -1,2 +0,0 @@
-from termforces.cmds.session_manager import *
-from termforces.cmds.submit_interface import *

termforces/cmds/session_manager.py

@@ -1,43 +0,0 @@
-import click
-import os
-from termforces.termforces_shell import termforces_shell, State
-from termforces.session_manager import load_session, store_session
-from getpass import getpass
-from codeforces_scraper import ScraperError
-
-
-@termforces_shell.command()
-@click.argument('handle')
-@click.option('--password',
-              help="Will be prompted, so do not pass, unless you're sure for safety")
-@click.option('--session-file', help='Tries to store session on successful login')
-@click.option('--no-getpass', is_flag=True,
-              show_default=True, default=False, help='Read password from stdin instead of using getpass')
-@click.pass_obj
-def login(state: State, handle, password, session_file, no_getpass):
-    if password is None:
-        if no_getpass:
-            password = input()
-        else:
-            password = getpass(f'Codeforces password for {handle}: ')
-    try:
-        state.scraper.login(handle, password)
-    except ScraperError:
-        print('Failed to login, check your credentials')
-        return
-    if session_file is not None:
-        store_session(state.scraper, os.path.expanduser(session_file))
-
-
-@termforces_shell.command(name='load-session')
-@click.argument('session_file')
-@click.pass_obj
-def load_session_cmd(state: State, session_file):
-    load_session(state.scraper, os.path.expanduser(session_file))
-
-
-@termforces_shell.command(name='whoami')
-@click.pass_obj
-def whoami(state: State):
-    state.scraper.update_current_user()
-    print(state.scraper.current_user)

termforces/cmds/submit_interface.py

@@ -1,64 +0,0 @@
-import click
-from termforces.termforces_shell import termforces_shell, State
-from codeforces_scraper import Verdict
-from codeforces_scraper.languages import some_compiler_by_ext
-from termforces.utils import tcolors, str_from_timestamp
-
-
-@termforces_shell.command(name='enter-contest')
-@click.argument('contest_id')
-@click.pass_obj
-def enter_contest(state: State, contest_id: int):
-    state.contest_id = contest_id
-
-
-@termforces_shell.command(name='submit')
-@click.argument('problem_index')
-@click.argument('source_file')
-@click.option('--contest-id')
-@click.option('--lang-code')
-@click.pass_obj
-def submit(state: State, problem_index, source_file, contest_id, lang_code):
-    if contest_id is None:
-        if state.contest_id is not None:
-            contest_id = state.contest_id
-        else:
-            print('Specify contest id or enter contest via enter-contest command')
-            return
-    if lang_code is None:
-        ext = '.' + source_file.split('.')[-1]
-        compiler = some_compiler_by_ext(ext)
-        if compiler is None:
-            print('Please specify language code')
-            return
-        lang_code = compiler.id
-    with open(source_file, 'r') as f:
-        source_code = f.read()
-    state.scraper.submit(contest_id, problem_index, source_code, lang_code)
-
-
-@termforces_shell.command(name='results-my')
-@click.option('--contest-id')
-@click.pass_obj
-def results_my(state: State, contest_id):
-    if contest_id is None:
-        if state.contest_id is not None:
-            contest_id = state.contest_id
-        else:
-            print('Specify contest id or enter contest via enter-contest command')
-            return
-    if state.scraper.current_user is None:
-        print('Please login to view your results')
-        return
-    subms = state.scraper.get_submissions(contest_id, state.scraper.current_user)
-    subms.sort(key=lambda subm: (subm.problem.index, subm.creation_time_seconds))
-
-    for subm in subms:
-        if subm.verdict == Verdict.TESTING:
-            color = tcolors.WARNING
-        elif subm.verdict == Verdict.OK:
-            color = tcolors.OKGREEN
-        else:
-            color = tcolors.FAIL
-        verdict_str = f'{color}{subm.verdict.name}{tcolors.ENDC}'
-        print(f'{subm.problem.index} \t {str_from_timestamp(subm.creation_time_seconds)} \t {verdict_str}')

termforces/session_manager.py

@@ -1,23 +0,0 @@
-import json
-import requests
-from codeforces_scraper import Scraper
-
-
-# TODO: it should be scraper method
-def store_session(scraper: Scraper, session_file):
-    with open(session_file, 'w') as f:
-        d = requests.utils.dict_from_cookiejar(scraper.session.cookies)
-        d['__termforces_name'] = scraper.current_user
-        json.dump(d, f)
-
-
-# TODO: it should be a scraper method
-def load_session(scraper: Scraper, session_file):
-    with open(session_file, 'r') as f:
-        d = json.load(f)
-        name = d['__termforces_name']
-        del d['__termforces_name']
-        new_cookies = requests.utils.cookiejar_from_dict(d)
-        scraper.current_user = name
-        scraper.session.cookies.clear()
-        scraper.session.cookies.update(new_cookies)

termforces/termforces_shell.py

@@ -1,31 +0,0 @@
-import click
-import os
-from click_shell import shell
-from codeforces_scraper import Scraper
-from termforces.session_manager import load_session
-
-SEARCH_LOCATIONS = ['.', '..', os.path.join(os.path.expanduser('~'), '.config', 'termforces')]
-FILE_NAME = 'termforces_cookies.json'
-
-
-class State:
-    def __init__(self):
-        self.scraper = Scraper()
-
-
-def preload_session(state):
-    for loc in SEARCH_LOCATIONS:
-        file_path = os.path.join(loc, FILE_NAME)
-        if os.path.exists(file_path):
-            print(f'Loading session from {file_path}')
-            load_session(state.scraper, file_path)
-            return
-
-
-@shell(prompt='termforces > ', intro='Entering termforces shell')
-@click.pass_context
-def termforces_shell(ctx):
-    state = State()
-    preload_session(state)
-    ctx.obj = state
-    pass

termforces/utils.py

@@ -1,18 +0,0 @@
-from datetime import datetime
-
-
-class tcolors:
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKCYAN = '\033[96m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-
-def str_from_timestamp(timestamp: int):
-    date = datetime.fromtimestamp(timestamp)
-    return date.strftime('%d.%m.%y %T')