README.md

@@ -28,3 +28,22 @@ python3 setup.py install
 
 ## Reference
 A comment provided for each function.
+
+## Termforces
+Termforces is a CLI wrapper for scraper. Currently it has a python module and a shell script
+
+## Quickstart
+Use `termforces login <username>` to login, it will store your session,
+so you're not needed to do it often.
+
+Then enter a directory you want and use `termforces strap --contest-id <contest_id> --indices <indices>`.
+
+Indices should be separated with space, i.e `termforces strap --contest-id 1329 --indices "A B1 B2 C D E"`.
+It will create folders `problem$index` for each index you specified.
+
+You may also specify template folder with `--template <path-to-template-folder>`, in this case script will copy
+its contents to all problem subfolders.
+
+You can check your results with `termforces results` and submit files with `termforces submit <filename>`.
+Script will determine contest id from parent .rc file and problem index from folder name. You may run it either from parent or from child
+directory.

codeforces_scraper/models.py

@@ -71,8 +71,8 @@ class APIModel(BaseModel):
 
 class JudgeProtocol(APIModel):
     manual: bool
-    protocol: Optional[str] = None
+    protocol: Optional[str]
-    verdict: Optional[str] = None
+    verdict: Optional[str]
 
 
 class BlogEntry(APIModel):
@@ -81,7 +81,7 @@ class BlogEntry(APIModel):
     creation_time_seconds: int
     author_handle: str
     title: str
-    content: Optional[str] = None
+    content: Optional[str]
     locale: str
     modification_time_seconds: int
     allow_view_history: bool
@@ -95,7 +95,7 @@ class Comment(APIModel):
     commentator_handle: str
     locale: str
     text: str
-    parent_comment_id: Optional[int] = None
+    parent_comment_id: Optional[int]
     rating: int
 
 
@@ -126,26 +126,26 @@ class Member(APIModel):
 
 
 class Problem(APIModel):
-    contest_id: Optional[int] = None
+    contest_id: Optional[int]
-    problem_set_name: Optional[str] = None
+    problem_set_name: Optional[str]
     index: str
     name: str
     type: str
-    points: Optional[float] = None
+    points: Optional[float]
-    rating: Optional[int] = None
+    rating: Optional[int]
     tags: List[str]
 
 
 class User(APIModel):
     handle: str
-    email: Optional[str] = None
+    email: Optional[str]
-    vk_id: Optional[str] = None
+    vk_id: Optional[str]
-    open_id: Optional[str] = None
+    open_id: Optional[str]
-    first_name: Optional[str] = None
+    first_name: Optional[str]
-    last_name: Optional[str] = None
+    last_name: Optional[str]
-    country: Optional[str] = None
+    country: Optional[str]
-    city: Optional[str] = None
+    city: Optional[str]
-    organization: Optional[str] = None
+    organization: Optional[str]
     contribution: int
     rank: str
     rating: int
@@ -162,11 +162,11 @@ class Party(APIModel):
     contest_id: int
     members: List[Member]
     participant_type: str
-    team_id: Optional[int] = None
+    team_id: Optional[int]
-    team_name: Optional[str] = None
+    team_name: Optional[str]
     ghost: bool
-    room: Optional[int] = None
+    room: Optional[int]
-    start_time_seconds: Optional[int] = None
+    start_time_seconds: Optional[int]
 
 
 class Submission(APIModel):
@@ -177,12 +177,12 @@ class Submission(APIModel):
     problem: Problem
     author: Party
     programming_language: str
-    verdict: Optional[Verdict] = None
+    verdict: Optional[Verdict]
     testset: str
     passed_test_count: int
     time_consumed_millis: int
     memory_consumed_bytes: int
-    points: Optional[float] = None
+    points: Optional[float]
 
 
 class Contest(APIModel):
@@ -192,17 +192,17 @@ class Contest(APIModel):
     phase: ContestPhase
     frozen: bool
     duration_seconds: bool
-    start_time_seconds: Optional[int] = None
+    start_time_seconds: Optional[int]
-    relative_time_seconds: Optional[int] = None
+    relative_time_seconds: Optional[int]
-    prepared_by: Optional[str] = None
+    prepared_by: Optional[str]
-    website_url: Optional[str] = None
+    website_url: Optional[str]
-    description: Optional[str] = None
+    description: Optional[str]
-    difficulty: Optional[int] = None
+    difficulty: Optional[int]
-    kind: Optional[str] = None
+    kind: Optional[str]
-    icpc_region: Optional[str] = None
+    icpc_region: Optional[str]
-    country: Optional[str] = None
+    country: Optional[str]
-    city: Optional[str] = None
+    city: Optional[str]
-    season: Optional[str] = None
+    season: Optional[str]
 
 
 class Hack(APIModel):
@@ -211,8 +211,8 @@ class Hack(APIModel):
     hacker: Party
     defender: Party
     problem: Problem
-    test: Optional[str] = None
+    test: Optional[str]
-    judge_protocol: JudgeProtocol
+    judge_protocol = JudgeProtocol
 
 
 class ProblemResult(APIModel):
@@ -232,8 +232,3 @@ class RanklistRow(APIModel):
     unsuccessful_hack_count: int
     problem_result: List[ProblemResult]
     last_submission_time_seconds: int
-
-
-class Sample(BaseModel):
-    s_in: str
-    s_out: str

codeforces_scraper/scraper.py

@@ -3,8 +3,8 @@ import requests
 from requests import Session
 from bs4 import BeautifulSoup as bs
 
-from codeforces_scraper.utils import get_token, get_messages, create_jar, unfuck_multitest_sample
+from codeforces_scraper.utils import get_token, get_messages, create_jar
-from codeforces_scraper.models import Submission, Problem, Sample
+from codeforces_scraper.models import Submission, Problem
 from typing import List
 
 
@@ -62,7 +62,7 @@ class Scraper:
                 if self.current_user is not None:
                     raise ScraperError('Failed to logout!')
                 return
-
+    
     def get_csrf_token(self):
         """Get csrf token, which is needed
         to make requests by hand
@@ -114,10 +114,8 @@ class Scraper:
             raise ScraperError('Submitting while not logged in')
         url = f'contest/{contest_id}/submit'
         submit_page_response = self.get(url)
-        # FIXME: Now some pornography is in the messages, which is not displayed and
+        for message in get_messages(submit_page_response):
-        # is not an error
+            raise MessagedScrapError(message)
-        # for message in get_messages(submit_page_response):
-        #     raise MessagedScrapError(message)
         token = get_token(submit_page_response)
         payload = {
             'csrf_token': token,
@@ -169,7 +167,7 @@ class Scraper:
             }
         else:
             params = {'contestId': contest_id}
-        return [Submission.model_validate(x) for x in self.api_request('contest.status', params)]
+        return [Submission.parse_obj(x) for x in self.api_request('contest.status', params)]
 
     def get_contest_tasks(self, contest_id: int) -> List[Problem]:
         """Get all tasks in contest with id ``contest_id``"""
@@ -179,16 +177,6 @@ class Scraper:
         }
         return self.api_request('contest.standings', params)['problems']
 
-    def get_samples(self, contest_id: int, problem_index: str) -> List[Sample]:
-        url = f'contest/{contest_id}/problem/{problem_index}'
-        page_response = self.get(url)
-        soup = bs(page_response.text, 'lxml')
-        samples = soup.find(attrs={'class': 'sample-tests'}).find(attrs={'class': 'sample-test'})
-        inputs = [unfuck_multitest_sample(str(div_input.find(name='pre')))
-                  for div_input in samples.find_all(attrs={'class': 'input'})]
-        outputs = [div_output.find(name='pre').get_text() for div_output in samples.find_all(attrs={'class', 'output'})]
-        return [Sample(s_in=s_in, s_out=s_out) for (s_in, s_out) in zip(inputs, outputs)]
-
     def get(self, sub_url='', **kwargs):
         """Make a GET request to BASE_URL"""
         url = self.base_url + '/' + sub_url

codeforces_scraper/utils.py

@@ -8,15 +8,6 @@ MESSAGE_GREP_STRING = r'Codeforces\.showMessage\('
 # TODO: Grep for Codeforces.showMessage(" to find message, that has been sent
 
 
-def unfuck_multitest_sample(sample_input: str) -> str:
-    div_class_regex = '<div class="[a-zA-Z0-9- ]*">'
-    sample_input = re.sub(div_class_regex, '', sample_input)
-    sample_input = re.sub('</div>', '\n', sample_input)
-    sample_input = re.sub('<pre>', '', sample_input)
-    sample_input = re.sub('</pre>', '', sample_input)
-    return sample_input
-
-
 def create_jar(str_cookie: str):
     cookies = str_cookie.split(';')
     d = {}
@@ -36,7 +27,6 @@ def get_token(response: Response) -> str:
     return token
 
 
-# FIXME: More robust way to find messages
 def get_messages(response: Response) -> List[str]:
     text = response.text
     return re.findall(fr'{MESSAGE_GREP_STRING}\"(.+?)\"', text)

scripts/termforces

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Minimalistic script to interact with Codeforces
+# Written by thematdev in 2023
+
+
+load_rc() {
+    if [[ -e $1 ]]; then source $1; fi;
+}
+
+# Load .rc files
+load_rc ~/.config/termforces/termforces.rc
+load_rc ../termforces.rc
+load_rc termforces.rc
+
+login() {
+    if [[ $TF_PASS_CMD ]]
+    then
+        python3 -m termforces login "$1" --session-file "~/.config/termforces/termforces_cookies.json" --no-getpass <<< "$($TF_PASS_CMD)"
+    else
+        python3 -m termforces login "$1" --session-file "~/.config/termforces/termforces_cookies.json"
+    fi
+}
+
+whoami() {
+    python3 -m termforces whoami
+}
+
+submit() {
+    source_file=$1
+    # Calling from either parent or child directory
+    if [ $(dirname $1) = "." ]; then
+        p=$(pwd)
+    else
+        p=$(dirname $1)
+    fi
+    problem_index=${p:0-1}
+    python3 -m termforces submit --contest-id $contest_id $problem_index $source_file
+}
+
+results() {
+    python3 -m termforces results-my --contest-id $contest_id
+}
+
+strap() {
+    while [[ $# -gt 0 ]]
+    do
+        key="$1"
+        case $key in
+            -c|--contest-id)
+                contest_id="$2"
+                shift
+                shift
+                ;;
+            -i|--indices)
+                indices="$2"
+                shift
+                shift
+                ;;
+            -t|--template)
+                template="$2"
+                shift
+                shift
+                ;;
+            *)
+                POSITIONAL+=("$1")
+                shift
+                ;;
+        esac
+    done
+    read -a indices_arr <<< $indices
+    if [[ ! -d $template ]]; then echo "Template directory not found or not specified, will create empty"; fi;
+    for index in "${indices_arr[@]}"
+    do
+        if [[ -d $template ]]
+        then
+            cp -r $template problem$index
+        else
+            mkdir problem$index
+        fi
+    done
+    echo "contest_id=$contest_id" >> termforces.rc
+}
+
+"$@"

setup.py

@@ -2,16 +2,20 @@ import setuptools
 
 setuptools.setup(
         name='codeforces-scraper',
-        version='0.4.0',
+        version='0.2.0-r1',
         author='thematdev',
         author_email='thematdev@thematdev.org',
         description='Utility to do actions on codeforces',
-        packages=['codeforces_scraper', 'codeforces_scraper.assets'],
+        packages=['codeforces_scraper', 'codeforces_scraper.assets',
+                  'termforces', 'termforces.cmds'],
+        scripts=['scripts/termforces'],
         install_requires=[
             'bs4',
             'lxml',
             'pydantic',
             'requests',
+            'click',
+            'click_shell'
         ],
         python_requires='>=3.8',
         zip_safe=True,

termforces/__main__.py

@@ -0,0 +1,6 @@
+from termforces.termforces_shell import termforces_shell
+from termforces.cmds import *
+
+
+if __name__ == '__main__':
+    termforces_shell()

termforces/cmds/__init__.py

@@ -0,0 +1,2 @@
+from termforces.cmds.session_manager import *
+from termforces.cmds.submit_interface import *

termforces/cmds/session_manager.py

@@ -0,0 +1,43 @@
+import click
+import os
+from termforces.termforces_shell import termforces_shell, State
+from termforces.session_manager import load_session, store_session
+from getpass import getpass
+from codeforces_scraper import ScraperError
+
+
+@termforces_shell.command()
+@click.argument('handle')
+@click.option('--password',
+              help="Will be prompted, so do not pass, unless you're sure for safety")
+@click.option('--session-file', help='Tries to store session on successful login')
+@click.option('--no-getpass', is_flag=True,
+              show_default=True, default=False, help='Read password from stdin instead of using getpass')
+@click.pass_obj
+def login(state: State, handle, password, session_file, no_getpass):
+    if password is None:
+        if no_getpass:
+            password = input()
+        else:
+            password = getpass(f'Codeforces password for {handle}: ')
+    try:
+        state.scraper.login(handle, password)
+    except ScraperError:
+        print('Failed to login, check your credentials')
+        return
+    if session_file is not None:
+        store_session(state.scraper, os.path.expanduser(session_file))
+
+
+@termforces_shell.command(name='load-session')
+@click.argument('session_file')
+@click.pass_obj
+def load_session_cmd(state: State, session_file):
+    load_session(state.scraper, os.path.expanduser(session_file))
+
+
+@termforces_shell.command(name='whoami')
+@click.pass_obj
+def whoami(state: State):
+    state.scraper.update_current_user()
+    print(state.scraper.current_user)

termforces/cmds/submit_interface.py

@@ -0,0 +1,64 @@
+import click
+from termforces.termforces_shell import termforces_shell, State
+from codeforces_scraper import Verdict
+from codeforces_scraper.languages import some_compiler_by_ext
+from termforces.utils import tcolors, str_from_timestamp
+
+
+@termforces_shell.command(name='enter-contest')
+@click.argument('contest_id')
+@click.pass_obj
+def enter_contest(state: State, contest_id: int):
+    state.contest_id = contest_id
+
+
+@termforces_shell.command(name='submit')
+@click.argument('problem_index')
+@click.argument('source_file')
+@click.option('--contest-id')
+@click.option('--lang-code')
+@click.pass_obj
+def submit(state: State, problem_index, source_file, contest_id, lang_code):
+    if contest_id is None:
+        if state.contest_id is not None:
+            contest_id = state.contest_id
+        else:
+            print('Specify contest id or enter contest via enter-contest command')
+            return
+    if lang_code is None:
+        ext = '.' + source_file.split('.')[-1]
+        compiler = some_compiler_by_ext(ext)
+        if compiler is None:
+            print('Please specify language code')
+            return
+        lang_code = compiler.id
+    with open(source_file, 'r') as f:
+        source_code = f.read()
+    state.scraper.submit(contest_id, problem_index, source_code, lang_code)
+
+
+@termforces_shell.command(name='results-my')
+@click.option('--contest-id')
+@click.pass_obj
+def results_my(state: State, contest_id):
+    if contest_id is None:
+        if state.contest_id is not None:
+            contest_id = state.contest_id
+        else:
+            print('Specify contest id or enter contest via enter-contest command')
+            return
+    if state.scraper.current_user is None:
+        print('Please login to view your results')
+        return
+    subms = state.scraper.get_submissions(contest_id, state.scraper.current_user)
+    subms.sort(key=lambda subm: (subm.problem.index, subm.creation_time_seconds))
+
+    for subm in subms:
+        if subm.verdict == Verdict.TESTING:
+            color = tcolors.WARNING
+        elif subm.verdict == Verdict.OK:
+            color = tcolors.OKGREEN
+        else:
+            color = tcolors.FAIL
+        verdict_str = f'{color}{subm.verdict.name}{tcolors.ENDC}'
+        print(f'{subm.problem.index} \t {str_from_timestamp(subm.creation_time_seconds)} \t {verdict_str}')

termforces/session_manager.py

@@ -0,0 +1,23 @@
+import json
+import requests
+from codeforces_scraper import Scraper
+
+
+# TODO: it should be scraper method
+def store_session(scraper: Scraper, session_file):
+    with open(session_file, 'w') as f:
+        d = requests.utils.dict_from_cookiejar(scraper.session.cookies)
+        d['__termforces_name'] = scraper.current_user
+        json.dump(d, f)
+
+
+# TODO: it should be a scraper method
+def load_session(scraper: Scraper, session_file):
+    with open(session_file, 'r') as f:
+        d = json.load(f)
+        name = d['__termforces_name']
+        del d['__termforces_name']
+        new_cookies = requests.utils.cookiejar_from_dict(d)
+        scraper.current_user = name
+        scraper.session.cookies.clear()
+        scraper.session.cookies.update(new_cookies)

termforces/termforces_shell.py

@@ -0,0 +1,31 @@
+import click
+import os
+from click_shell import shell
+from codeforces_scraper import Scraper
+from termforces.session_manager import load_session
+
+SEARCH_LOCATIONS = ['.', '..', os.path.join(os.path.expanduser('~'), '.config', 'termforces')]
+FILE_NAME = 'termforces_cookies.json'
+
+
+class State:
+    def __init__(self):
+        self.scraper = Scraper()
+
+
+def preload_session(state):
+    for loc in SEARCH_LOCATIONS:
+        file_path = os.path.join(loc, FILE_NAME)
+        if os.path.exists(file_path):
+            print(f'Loading session from {file_path}')
+            load_session(state.scraper, file_path)
+            return
+
+
+@shell(prompt='termforces > ', intro='Entering termforces shell')
+@click.pass_context
+def termforces_shell(ctx):
+    state = State()
+    preload_session(state)
+    ctx.obj = state
+    pass

termforces/utils.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+
+
+class tcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+def str_from_timestamp(timestamp: int):
+    date = datetime.fromtimestamp(timestamp)
+    return date.strftime('%d.%m.%y %T')