From 32db5e7870d0041f064392427acfe9690b5e2727 Mon Sep 17 00:00:00 2001 From: PhoenixTamaoki Date: Tue, 5 Apr 2022 16:41:55 -0400 Subject: [PATCH 01/13] converted course_corequisite --- src/api/models/course_corequisite.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/api/models/course_corequisite.py diff --git a/src/api/models/course_corequisite.py b/src/api/models/course_corequisite.py new file mode 100644 index 000000000..8d59be2ed --- /dev/null +++ b/src/api/models/course_corequisite.py @@ -0,0 +1,11 @@ +from tortoise import fields +from tortoise.models import Model + +class CourseCorequisite(Model): + + department = fields.IntField(length=255, pk = True) + level = fields.IntField(pk = True) + corequisite = fields.IntField(length=255, pk = True) + + class Meta: + table = "course_corequisite" \ No newline at end of file From 303612285c5ecc6e11555919f995d24c60791453 Mon Sep 17 00:00:00 2001 From: PhoenixTamaoki Date: Tue, 5 Apr 2022 16:42:23 -0400 Subject: [PATCH 02/13] converted course_corequisite --- src/api/models/course_corequisite.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/api/models/course_corequisite.py b/src/api/models/course_corequisite.py index 8d59be2ed..ae002b657 100644 --- a/src/api/models/course_corequisite.py +++ b/src/api/models/course_corequisite.py @@ -3,9 +3,10 @@ class CourseCorequisite(Model): - department = fields.IntField(length=255, pk = True) - level = fields.IntField(pk = True) - corequisite = fields.IntField(length=255, pk = True) + department = fields.IntField(length=255) + level = fields.IntField() + corequisite = fields.IntField(length=255) class Meta: - table = "course_corequisite" \ No newline at end of file + table = "course_corequisite" + unique_together = (('department', 'level', 'corequisite'),) \ No newline at end of file From 71d2347851002655eddf3efe4c0bec82563229b4 Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Wed, 6 Apr 2022 16:06:02 -0400 Subject: [PATCH 03/13] updated course model --- src/api/models/course.py | 29 +++++++++++++++++++++++++++++ src/api/tables/course.py | 27 --------------------------- 2 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 src/api/models/course.py delete mode 100644 src/api/tables/course.py diff --git a/src/api/models/course.py b/src/api/models/course.py new file mode 100644 index 000000000..73d7c5b10 --- /dev/null +++ b/src/api/models/course.py @@ -0,0 +1,29 @@ +from tortoise import fields +from tortoise.models import Model + + +class Course(Model): + __tablename__ = "course" + + crn = fields.CharField(max_length=255, primary_key=True) + section = fields.CharField(max_length=255) + semester = fields.CharField(max_length=255) + min_credits = fields.IntField() + max_credits = fields.IntField() + date_start = fields.DatetimeField() + date_end = fields.DatetimeField() + department = fields.CharField(max_length=255) + level = fields.IntField() + title = fields.CharField(max_length=255) + full_title = fields.TextField() + description = fields.TextField() + raw_precoreqs = fields.TextField() + frequency = fields.CharField(max_length=255) + school = fields.CharField(max_length=255) + seats_open = fields.IntField() + seats_filled = fields.IntField() + seats_total = fields.IntField() + tsv = fields.TSVectorField() + + class Meta: + table = "course" \ No newline at end of file diff --git a/src/api/tables/course.py b/src/api/tables/course.py deleted file mode 100644 index 7e42fc88f..000000000 --- a/src/api/tables/course.py +++ /dev/null @@ -1,27 +0,0 @@ -from sqlalchemy import Column, PrimaryKeyConstraint -from sqlalchemy.dialects.postgresql import TEXT, INTEGER, VARCHAR, DATE, TSVECTOR - -from .database import Base - -class Course(Base): - __tablename__ = "course" - - crn = Column(VARCHAR(length=255), primary_key=True) - section = Column(VARCHAR(length=255)) - semester = Column(VARCHAR(length=255)) - min_credits = Column(INTEGER) - max_credits = Column(INTEGER) - date_start = Column(DATE) - date_end = Column(DATE) - department = Column(VARCHAR(length=255)) - level = Column(INTEGER) - title = Column(VARCHAR(length=255)) - full_title = Column(TEXT) - description = Column(TEXT) - raw_precoreqs = Column(TEXT) - frequency = Column(VARCHAR(length=255)) - school = Column(VARCHAR(length=255)) - seats_open = Column(INTEGER) - seats_filled = Column(INTEGER) - seats_total = Column(INTEGER) - tsv = Column(TSVECTOR) \ No newline at end of file From ac003e39caf71dff1d8aaa521107837735725eb2 Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 16:19:07 -0400 Subject: [PATCH 04/13] fixed for pytests --- src/api/models/__init__.py | 3 ++- src/api/tables/__init__.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/models/__init__.py b/src/api/models/__init__.py index 3f5b70e4e..02773920e 100644 --- a/src/api/models/__init__.py +++ b/src/api/models/__init__.py @@ -1,3 +1,4 @@ from .user_account import UserAccount from .user_session import UserSession -from .event import Event \ No newline at end of file +from .event import Event +from .course import Course \ No newline at end of file diff --git a/src/api/tables/__init__.py b/src/api/tables/__init__.py index 02a8e5e6a..1b1b44447 100644 --- a/src/api/tables/__init__.py +++ b/src/api/tables/__init__.py @@ -2,7 +2,6 @@ from .course_corequisite import CourseCorequisite from .course_prerequisite import CoursePrerequisite from .course_session import CourseSession -from .course import Course from .semester_date_range import SemesterDateRange from .semester_info import SemesterInfo from .student_course_selection import StudentCourseSelection From 4737dd58b04cd69b29d7e2d5bd5e015cdc4368a7 Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 16:26:46 -0400 Subject: [PATCH 05/13] fixed for pytests --- src/api/models/course.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/models/course.py b/src/api/models/course.py index 73d7c5b10..ea20e0f2a 100644 --- a/src/api/models/course.py +++ b/src/api/models/course.py @@ -23,7 +23,7 @@ class Course(Model): seats_open = fields.IntField() seats_filled = fields.IntField() seats_total = fields.IntField() - tsv = fields.TSVectorField() + tsv = tortoise.contrib.postgres.fields.TSVectorField() class Meta: table = "course" \ No newline at end of file From 82f2ef34cb903304c11612372ea87bf89fee5220 Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 16:29:00 -0400 Subject: [PATCH 06/13] workflows --- .github/workflows/ci.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2ebf80dd8..448d2a4d5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -62,16 +62,6 @@ jobs: run: | bash src/api/tests/test.sh - - name: Run API unit tests - env: - DB_PORT: 5432 - DB_USER: postgres - DB_PASS: postgres - DB_NAME: yacs - DB_HOST: postgres - run: | - bash src/api/tests/test.sh - lighthouse: name: Lighthouse CI runs-on: ubuntu-latest From 100206dd07049783da6d4721da1f3fb46fd91e8f Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 16:31:25 -0400 Subject: [PATCH 07/13] fixed for pytests --- src/api/models/course.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/models/course.py b/src/api/models/course.py index ea20e0f2a..41f4ad699 100644 --- a/src/api/models/course.py +++ b/src/api/models/course.py @@ -1,5 +1,6 @@ from tortoise import fields from tortoise.models import Model +import tortoise class Course(Model): From b1fa889adcd4c94335626e9476e4da2a5069ef08 Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 16:34:53 -0400 Subject: [PATCH 08/13] fixed for pytests --- src/api/models/course.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/models/course.py b/src/api/models/course.py index 41f4ad699..05b667f2a 100644 --- a/src/api/models/course.py +++ b/src/api/models/course.py @@ -1,6 +1,6 @@ from tortoise import fields from tortoise.models import Model -import tortoise +from tortoise.contrip.postgres.fields import TSVectorField class Course(Model): @@ -24,7 +24,7 @@ class Course(Model): seats_open = fields.IntField() seats_filled = fields.IntField() seats_total = fields.IntField() - tsv = tortoise.contrib.postgres.fields.TSVectorField() + tsv = TSVectorField() class Meta: table = "course" \ No newline at end of file From b233fad9d737524d46f77ef7ea08ca67ae37fbba Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 17:26:11 -0400 Subject: [PATCH 09/13] pytest fixings --- src/api/models/course.py | 5 +++-- src/api/requirements.txt | 1 + src/api/tests/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/api/models/course.py b/src/api/models/course.py index 05b667f2a..1aadf1662 100644 --- a/src/api/models/course.py +++ b/src/api/models/course.py @@ -1,6 +1,7 @@ +import tortoise +from tortoise.contrib.postgres import fields as pfields from tortoise import fields from tortoise.models import Model -from tortoise.contrip.postgres.fields import TSVectorField class Course(Model): @@ -24,7 +25,7 @@ class Course(Model): seats_open = fields.IntField() seats_filled = fields.IntField() seats_total = fields.IntField() - tsv = TSVectorField() + tsv = pfields.TSVectorField() class Meta: table = "course" \ No newline at end of file diff --git a/src/api/requirements.txt b/src/api/requirements.txt index fc7a26f34..6a16afee9 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -10,3 +10,4 @@ fastapi-cache2==0.1.8 itsdangerous==2.0.1 tortoise-orm==0.19.0 asyncpg==0.25.0 +requests==2.27.1 diff --git a/src/api/tests/Dockerfile b/src/api/tests/Dockerfile index 825ef3cdf..61d1e90b8 100644 --- a/src/api/tests/Dockerfile +++ b/src/api/tests/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.8-slim RUN mkdir -p /usr/src WORKDIR /usr/src COPY ./../requirements.txt /usr/src/ -RUN pip install --no-cache-dir -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements.txt COPY ../ /usr/src/ CMD [ "sh", "tests/test.sh" ] From 9a7c2728df7a71672f8a4ed1de3bd513c6420745 Mon Sep 17 00:00:00 2001 From: PhoenixTamaoki Date: Fri, 8 Apr 2022 17:31:14 -0400 Subject: [PATCH 10/13] forgot to add last changes --- src/api/models/__init__.py | 3 +- src/api/tables/course_corequisite.py | 15 --------- tests/api/db/test_saving.py | 48 ---------------------------- 3 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 src/api/tables/course_corequisite.py delete mode 100644 tests/api/db/test_saving.py diff --git a/src/api/models/__init__.py b/src/api/models/__init__.py index 512995484..4734d6c7c 100644 --- a/src/api/models/__init__.py +++ b/src/api/models/__init__.py @@ -1,2 +1,3 @@ from .user_account import UserAccount -from .user_session import UserSession \ No newline at end of file +from .user_session import UserSession +from .course_corequisite import CourseCorequisite diff --git a/src/api/tables/course_corequisite.py b/src/api/tables/course_corequisite.py deleted file mode 100644 index e5f29485b..000000000 --- a/src/api/tables/course_corequisite.py +++ /dev/null @@ -1,15 +0,0 @@ -from sqlalchemy import Column, PrimaryKeyConstraint -from sqlalchemy.dialects.postgresql import VARCHAR, INTEGER - -from .database import Base - -class CourseCorequisite(Base): - __tablename__ = 'course_corequisite' - - department = Column(VARCHAR(length=255)) - level = Column(INTEGER) - corequisite = Column(VARCHAR(length=255)) - - __table_args__ = ( - PrimaryKeyConstraint('department', 'level', 'corequisite'), - ) \ No newline at end of file diff --git a/tests/api/db/test_saving.py b/tests/api/db/test_saving.py deleted file mode 100644 index 6e916b772..000000000 --- a/tests/api/db/test_saving.py +++ /dev/null @@ -1,48 +0,0 @@ -def test_save_student_courses(): ##save_semester, test_user): - assert(True) - - ## Code for future testing of semester saving system - # user = {"name":"Barb", "email":"user1@rpi.edu", "phone":"1234567890", "major":"CS", "password":"123456", "degree":"Undergraduate"} - # user1 = {"name":"Wes", "email":"user2@rpi.edu", "phone":"0123456789", "major":"CS", "password":"246810", "degree":"Undergraduate"} - - # uid = test_user.add_user(user) - # uid1 = test_user.add_user(user1) - # ##print(uid) - # ##print(uid1) - - # (status, err) = save_semester.add_selection('yacs-101', 'FALL 2020', uid, '-1') - # assert(err is None) - # (status1, err) = save_semester.add_selection('yacs-404', 'SPRING 2021', uid, '-1') - # assert(err is None) - # (status2, err) = save_semester.add_selection('rcos', 'FALL 2020', uid1, '-1') - # assert(err is None) - # (status3, err) = save_semester.add_selection('yacs-101', 'FALL 2020', uid, '12345') - # assert(err is None) - # (status4, err) = save_semester.add_selection('yacs-404', 'SPRING 2021', uid, '24680') - # assert(err is None) - # (status5, err) = save_semester.add_selection('rcos', 'FALL 2020', uid1, '12345') - # assert(err is None) - - # assert(status and status1 and status2 and status3 and status4 and status5) - # (selections, err) = get_selection(uid) - # assert(err is None) - # (selections1, err) = get_selection(uid1) - # assert(err is None) - - # assert(len(selections) == 4) - # assert(len(selections1) == 2) - - # (status, err) = save_semester.remove_selection('rcos', 'FALL 2020', uid1, '12345') - # assert(err is None) - # (status1, err) = save_semester.remove_selection('yacs-404', 'SPRING 2021', uid) - # assert(err is None) - - # assert(status and status1) - - # (selections, err) = get_selection(uid) - # assert(err is None) - # (selections1, err) = get_selection(uid1) - # assert(err is None) - - # assert(len(selections) == 2) - # assert(len(selections1) == 1) From 559e2bc32b3bd5eb40fa3ae029b8b4cef7f95d0f Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 17:37:17 -0400 Subject: [PATCH 11/13] pytest fixes --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index f327be54b..dd26d8ab3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,7 @@ pytest==6.2.5 pytest_check==0.3.8 regex==2020.4.4 -requests==2.26.0 +requests==2.27.1 lxml==4.6.5 pandas==1.3.3 beautifulsoup4==4.9.0 From fbe73c5f43bfd28aa922c0bc846a192dae0e0d2a Mon Sep 17 00:00:00 2001 From: Liam Haining Date: Fri, 8 Apr 2022 17:40:48 -0400 Subject: [PATCH 12/13] fixed tables init file --- src/api/tables/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/tables/__init__.py b/src/api/tables/__init__.py index b55cb5423..e394acf29 100644 --- a/src/api/tables/__init__.py +++ b/src/api/tables/__init__.py @@ -1,5 +1,4 @@ from .admin_settings import AdminSettings -from .course_corequisite import CourseCorequisite from .course_prerequisite import CoursePrerequisite from .course_session import CourseSession from .course import Course From b38cc2d34d90f043acf00c4cbbbfb0f374a6b12d Mon Sep 17 00:00:00 2001 From: canghiskhan <66562619+canghiskhan@users.noreply.github.com> Date: Mon, 11 Apr 2022 19:32:18 -0400 Subject: [PATCH 13/13] course session tortoise conversion first draft (#567) * course session tortoise conversion first draft * fixing some issue * fixed fields Co-authored-by: canghiskhan Co-authored-by: Liam Haining Co-authored-by: Liam Haining <12472949+lhain08@users.noreply.github.com> --- src/api/db/courses.py | 90 ++++++++++++++++---------------- src/api/models/__init__.py | 3 +- src/api/models/course_session.py | 18 +++++++ 3 files changed, 65 insertions(+), 46 deletions(-) create mode 100644 src/api/models/course_session.py diff --git a/src/api/db/courses.py b/src/api/db/courses.py index e56fdabb5..2dc884e80 100644 --- a/src/api/db/courses.py +++ b/src/api/db/courses.py @@ -35,15 +35,15 @@ def getDays(self, daySequenceStr): return set(filter( lambda day: day, re.split("(?:(M|T|W|R|F))", daySequenceStr))) - def delete_by_semester(self, semester): + async def delete_by_semester(self, semester): # clear cache so this semester does not come up again self.clear_cache() - return self.db.execute(""" + return await self.db.execute(""" BEGIN TRANSACTION; DELETE FROM course - WHERE semester=%(Semester)s; + WHERE semester='%(Semester)s'; DELETE FROM course_session - WHERE semester=%(Semester)s; + WHERE semester='%(Semester)s'; COMMIT; """, { "Semester": semester @@ -59,8 +59,8 @@ def bulk_delete(self, semesters): self.clear_cache() return None - def populate_from_csv(self, csv_text): - conn = self.db.get_connection() + async def populate_from_csv(self, csv_text): + conn = await self.db.get_connection() reader = csv.DictReader(csv_text) # for each course entry insert sections and course sessions with conn.cursor(cursor_factory=RealDictCursor) as transaction: @@ -84,15 +84,15 @@ def populate_from_csv(self, csv_text): instructor ) VALUES ( - NULLIF(%(CRN)s, ''), - NULLIF(%(Section)s, ''), - NULLIF(%(Semester)s, ''), - %(StartTime)s, - %(EndTime)s, - %(WeekDay)s, - NULLIF(%(Location)s, ''), - NULLIF(%(SessionType)s, ''), - NULLIF(%(Instructor)s, '') + NULLIF('%(CRN)s', ''), + NULLIF('%(Section)s', ''), + NULLIF('%(Semester)s', ''), + '%(StartTime)s', + '%(EndTime)s', + '%(WeekDay)s', + NULLIF('%(Location)s', ''), + NULLIF('%(SessionType)s', ''), + NULLIF('%(Instructor)s', '') ) ON CONFLICT DO NOTHING; """, @@ -134,30 +134,30 @@ def populate_from_csv(self, csv_text): tsv ) VALUES ( - NULLIF(%(CRN)s, ''), - NULLIF(%(Section)s, ''), - NULLIF(%(Semester)s, ''), - %(MinCredits)s, - %(MaxCredits)s, - NULLIF(%(Description)s, ''), - NULLIF(%(Frequency)s, ''), - NULLIF(%(FullTitle)s, ''), - %(StartDate)s, - %(EndDate)s, - NULLIF(%(Department)s, ''), - %(Level)s, - NULLIF(%(Title)s, ''), - NULLIF(%(RawPrecoreqText)s, ''), - %(School)s, - %(SeatsOpen)s, - %(SeatsFilled)s, - %(SeatsTotal)s, - setweight(to_tsvector(coalesce(%(FullTitle)s, '')), 'A') || - setweight(to_tsvector(coalesce(%(Title)s, '')), 'A') || - setweight(to_tsvector(coalesce(%(Department)s, '')), 'A') || - setweight(to_tsvector(coalesce(%(CRN)s, '')), 'A') || - setweight(to_tsvector(coalesce(%(Level)s, '')), 'B') || - setweight(to_tsvector(coalesce(%(Description)s, '')), 'D') + NULLIF('%(CRN)s', ''), + NULLIF('%(Section)s', ''), + NULLIF('%(Semester)s', ''), + '%(MinCredits)s', + '%(MaxCredits)s', + NULLIF('%(Description)s', ''), + NULLIF('%(Frequency)s', ''), + NULLIF('%(FullTitle)s', ''), + '%(StartDate)s', + '%(EndDate)s', + NULLIF('%(Department)s', ''), + '%(Level)s', + NULLIF('%(Title)s', ''), + NULLIF('%(RawPrecoreqText)s', ''), + '%(School)s', + '%(SeatsOpen)s', + '%(SeatsFilled)s', + '%(SeatsTotal)s', + setweight(to_tsvector(coalesce('%(FullTitle)s', '')), 'A') || + setweight(to_tsvector(coalesce('%(Title)s', '')), 'A') || + setweight(to_tsvector(coalesce('%(Department)s', '')), 'A') || + setweight(to_tsvector(coalesce('%(CRN)s', '')), 'A') || + setweight(to_tsvector(coalesce('%(Level)s', '')), 'B') || + setweight(to_tsvector(coalesce('%(Description)s', '')), 'D') ) ON CONFLICT DO NOTHING; """, @@ -197,9 +197,9 @@ def populate_from_csv(self, csv_text): prerequisite ) VALUES ( - NULLIF(%(Department)s, ''), - %(Level)s, - NULLIF(%(Prerequisite)s, '') + NULLIF('%(Department)s', ''), + '%(Level)s', + NULLIF('%(Prerequisite)s', '') ) ON CONFLICT DO NOTHING; """, @@ -220,9 +220,9 @@ def populate_from_csv(self, csv_text): corequisite ) VALUES ( - NULLIF(%(Department)s, ''), - %(Level)s, - NULLIF(%(Corequisite)s, '') + NULLIF('%(Department)s', ''), + '%(Level)s', + NULLIF('%(Corequisite)s', '') ) ON CONFLICT DO NOTHING; """, diff --git a/src/api/models/__init__.py b/src/api/models/__init__.py index 02773920e..6f9c22f61 100644 --- a/src/api/models/__init__.py +++ b/src/api/models/__init__.py @@ -1,4 +1,5 @@ from .user_account import UserAccount from .user_session import UserSession +from .course import Course +from .course_session import CourseSession from .event import Event -from .course import Course \ No newline at end of file diff --git a/src/api/models/course_session.py b/src/api/models/course_session.py new file mode 100644 index 000000000..6c6cf2d91 --- /dev/null +++ b/src/api/models/course_session.py @@ -0,0 +1,18 @@ +from tortoise import fields +from tortoise.models import Model + + +class CourseSession(Model): + crn = fields.CharField(max_length=255) + section = fields.CharField(max_length=255) + semester = fields.CharField(max_length=255) + time_start = fields.TimeField() + time_end = fields.TimeField() + day_of_week = fields.IntField() + location = fields.CharField(max_length=255) + session_type = fields.CharField(max_length=255) + instructor = fields.CharField(max_length=255) + + class Meta: + table = "course_session" + unique_together = (('crn', 'section', 'semester', 'day_of_week'),) \ No newline at end of file