Bug fixes for transfermarkt and fotmob

.github/workflows/main.yaml

@@ -6,6 +6,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, 3.10, 3.11, 3.12]
+
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

Changelog.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## 1.1.5 2024-03-28
+* Moved from requirements.txt to pyproject.toml
+* Fixed fotmob team colors and player stat issues in fm_match_data
+* Team id bug fixes for transfermarkt
+* Transfermarkt tm_player_data bug fix for previous seasons
+
+
 ## 1.1.4 2024-01-15
 * added fbref functions
 	* fb_player_match_logs_data

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 ian-shepherd
+Copyright (c) 2024 ian-shepherd
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "reus"
+version = "1.1.5"
+description = "A package that allows you to pull soccer statistics, player market values, and transfer information, primarily from FBref, Fotmob, and Transfermarkt"
+authors = ["ian-shepherd <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+pandas = "^2.2.0"
+numpy = "^1.26.4"
+requests = "^2.23.0"
+beautifulsoup4 = "^4.10.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

reus/fotmob/fm_match_data.py

@@ -376,33 +376,41 @@ def fm_match_data(
         "team_y": general.get("awayTeam").get("name"),
         "id_x": general.get("homeTeam").get("id"),
         "id_y": general.get("awayTeam").get("id"),
-        "color_x": general.get("teamColors").get("home")
-        if isinstance(general.get("teamColors"), dict)
-        else general.get("teamColors")[0].get("color"),
-        "color_y": general.get("teamColors").get("away")
-        if isinstance(general.get("teamColors"), dict)
-        else general.get("teamColors")[1].get("color"),
+        "color_x": (
+            general.get("teamColors").get("lightMode").get("home")
+            if isinstance(general.get("teamColors"), dict)
+            else general.get("teamColors")[0].get("color")
+        ),
+        "color_y": (
+            general.get("teamColors").get("lightMode").get("away")
+            if isinstance(general.get("teamColors"), dict)
+            else general.get("teamColors")[1].get("color")
+        ),
         "score_x": header.get("teams")[0].get("score"),
         "score_y": header.get("teams")[1].get("score"),
         "reason": header.get("status").get("reason").get("long"),
-        "highlights": matchFacts.get("highlights").get("url")
-        if matchFacts.get("highlights")
-        else None,
+        "highlights": (
+            matchFacts.get("highlights").get("url")
+            if matchFacts.get("highlights")
+            else None
+        ),
         "venue": matchFacts.get("infoBox").get("Stadium").get("name"),
         "city": matchFacts.get("infoBox").get("Stadium").get("city"),
         "country": matchFacts.get("infoBox").get("Stadium").get("country"),
         "lat": matchFacts.get("infoBox").get("Stadium").get("lat"),
         "long": matchFacts.get("infoBox").get("Stadium").get("long"),
         "referee": matchFacts.get("infoBox").get("Referee").get("text"),
         "attendance": matchFacts.get("infoBox").get("Attendance"),
-        "player_of_match": " ".join(
-            matchFacts.get("playerOfTheMatch").get("name").values()
-        )
-        if matchFacts.get("playerOfTheMatch")
-        else None,
-        "player_of_match_id": matchFacts.get("playerOfTheMatch").get("id")
-        if matchFacts.get("playerOfTheMatch")
-        else None,
+        "player_of_match": (
+            " ".join(matchFacts.get("playerOfTheMatch").get("name").values())
+            if matchFacts.get("playerOfTheMatch")
+            else None
+        ),
+        "player_of_match_id": (
+            matchFacts.get("playerOfTheMatch").get("id")
+            if matchFacts.get("playerOfTheMatch")
+            else None
+        ),
     }
 
     events = matchFacts.get("events")["events"]

reus/fotmob/util.py

@@ -1,10 +1,16 @@
-def extract_player_stats(stats):
+def extract_player_stats(stats):  # noqa: C901
     mydict = {}
 
     playerStats = stats[0].get("stats")
     tmp = {}
     for k, v in playerStats.items():
-        tmp[k] = v.get("value")
+        val = v["stat"].get("value")
+        total = v["stat"].get("total")
+        if total is not None:
+            tmp[k] = f"{val}/{total}"
+        else:
+            tmp[k] = val
+
     playerStats = tmp
 
     mydict["rating"] = playerStats.get("FotMob rating")
@@ -26,7 +32,12 @@ def extract_player_stats(stats):
         playerStatsAttack = stats[1].get("stats")
         tmp = {}
         for k, v in playerStatsAttack.items():
-            tmp[k] = v.get("value")
+            val = v["stat"].get("value")
+            total = v["stat"].get("total")
+            if total is not None:
+                tmp[k] = f"{val}/{total}"
+            else:
+                tmp[k] = val
         playerStatsAttack = tmp
 
         mydict["shot_accuracy"] = playerStatsAttack.get("Shot accuracy")
@@ -50,7 +61,12 @@ def extract_player_stats(stats):
         playerStatsDefense = stats[2].get("stats")
         tmp = {}
         for k, v in playerStatsDefense.items():
-            tmp[k] = v.get("value")
+            val = v["stat"].get("value")
+            total = v["stat"].get("total")
+            if total is not None:
+                tmp[k] = f"{val}/{total}"
+            else:
+                tmp[k] = val
         playerStatsDefense = tmp
 
         mydict["tackles_won"] = playerStatsDefense.get("Tackles won")
@@ -67,7 +83,12 @@ def extract_player_stats(stats):
         playerStatsDuels = stats[3].get("stats")
         tmp = {}
         for k, v in playerStatsDuels.items():
-            tmp[k] = v.get("value")
+            val = v["stat"].get("value")
+            total = v["stat"].get("total")
+            if total is not None:
+                tmp[k] = f"{val}/{total}"
+            else:
+                tmp[k] = val
         playerStatsDuels = tmp
 
         mydict["ground_duels_won"] = playerStatsDuels.get("Ground duels won")

reus/transfermarkt/tm_player_metadata.py

@@ -173,7 +173,7 @@ def tm_player_metadata(pageSoup=None, url: str = None) -> dict:
     meta_attributes = {
         "joined": "Joined:",
         "contracted": "Contract expires:",
-        "extension": "Date of last contract extension:",
+        "extension": "Last contract extension:",
     }
 
     for attr, pattern in meta_attributes.items():

reus/transfermarkt/tm_team_player_data.py

@@ -18,7 +18,7 @@ def _get_team_id_and_club(team_id, club):
     )
     filtered_df = df[filter_condition]
     club = filtered_df.transfermarkt_link.iloc[0]
-    team_id = int(filtered_df.transfermarkt.iloc[0])
+    team_id = int(float(filtered_df.transfermarkt.iloc[0]))
 
     return club, team_id
 

reus/transfermarkt/tm_team_staff.py

@@ -17,7 +17,7 @@ def _get_team_id_and_club(team_id, club):
     )
     filtered_df = df[filter_condition]
     club = filtered_df.transfermarkt_link.iloc[0]
-    team_id = int(filtered_df.transfermarkt.iloc[0])
+    team_id = int(float(filtered_df.transfermarkt.iloc[0]))
 
     return club, team_id
 

reus/transfermarkt/tm_team_staff_history.py

@@ -17,7 +17,7 @@ def _get_team_id_and_club(team_id, club):
     )
     filtered_df = df[filter_condition]
     club = filtered_df.transfermarkt_link.iloc[0]
-    team_id = int(filtered_df.transfermarkt.iloc[0])
+    team_id = int(float(filtered_df.transfermarkt.iloc[0]))
 
     return club, team_id
 

reus/transfermarkt/tm_team_transfers.py

@@ -18,7 +18,7 @@ def _get_team_id_and_club(team_id, club):
     )
     filtered_df = df[filter_condition]
     club = filtered_df.transfermarkt_link.iloc[0]
-    team_id = int(filtered_df.transfermarkt.iloc[0])
+    team_id = int(float(filtered_df.transfermarkt.iloc[0]))
 
     return club, team_id
 

setup.py

@@ -1,7 +1,7 @@
 import setuptools
 
-VERSION = "1.1.4"
-DESCRIPTION = "A package that allows you to soccer information"
+VERSION = "1.1.5"
+DESCRIPTION = "A package that allows you scrape soccer information"
 LONG_DESCRIPTION = """
     A package that allows you to pull soccer statistics, player market values, and
      transfer information, primarily from FBref, Fotmob, and Transfermarkt"""
@@ -13,7 +13,12 @@
     long_description=LONG_DESCRIPTION,
     url="https://github.com/ian-shepherd/reus",
     author="Ian Shepherd",
-    install_requires=["pandas", "numpy", "requests", "beautifulsoup4>=4.10.0"],
+    install_requires=[
+        "pandas>=2.2.0",
+        "numpy>=1.26.4",
+        "requests>=2.23.0",
+        "beautifulsoup4>=4.10.0",
+    ],
     keywords=["python", "fbref", "fotmob", "transfermarkt", "soccer", "football"],
     classifiers=[
         "Programming Language :: Python :: 3",

tests/fbref/data/pulisic_2019_2020_scouting.json

@@ -10,7 +10,7 @@
         "Red Cards": {"value": "0.00", "percentile": 54},
         "xG: Expected Goals": {"value": "0.47", "percentile": 93},
         "npxG: Non-Penalty xG": {"value": "0.47", "percentile": 96},
-        "xAG: Expected Assisted Goals": {"value": "0.19", "percentile": 66},
+        "xAG: Exp. Assisted Goals": {"value": "0.19", "percentile": 66},
         "npxG + xAG": {"value": "0.66", "percentile": 93},
         "Progressive Carries": {"value": "5.59", "percentile": 92},
         "Progressive Passes": {"value": "3.82", "percentile": 51},
@@ -49,7 +49,7 @@
         "Passes Attempted (Long)": {"value": "2.56", "percentile": 27},
         "Pass Completion % (Long)": {"value": "51.0", "percentile": 49},
         "Assists": {"value": "0.21", "percentile": 70},
-        "xAG: Expected Assisted Goals": {"value": "0.19", "percentile": 66},
+        "xAG: Exp. Assisted Goals": {"value": "0.19", "percentile": 66},
         "xA: Expected Assists": {"value": "0.14", "percentile": 51},
         "Key Passes": {"value": "1.57", "percentile": 53},
         "Passes into Final Third": {"value": "1.31", "percentile": 25},

tests/fbref/data/reus_2018_2019_scouting.json

@@ -10,7 +10,7 @@
         "Red Cards": {"value": "0.04", "percentile": 8},
         "xG: Expected Goals": {"value": "0.61", "percentile": 97},
         "npxG: Non-Penalty xG": {"value": "0.52", "percentile": 97},
-        "xAG: Expected Assisted Goals": {"value": "0.25", "percentile": 75},
+        "xAG: Exp. Assisted Goals": {"value": "0.25", "percentile": 75},
         "npxG + xAG": {"value": "0.77", "percentile": 96},
         "Progressive Carries": {"value": "2.99", "percentile": 50},
         "Progressive Passes": {"value": "4.86", "percentile": 76},
@@ -49,7 +49,7 @@
         "Passes Attempted (Long)": {"value": "3.26", "percentile": 36},
         "Pass Completion % (Long)": {"value": "76.2", "percentile": 96},
         "Assists": {"value": "0.31", "percentile": 82},
-        "xAG: Expected Assisted Goals": {"value": "0.25", "percentile": 75},
+        "xAG: Exp. Assisted Goals": {"value": "0.25", "percentile": 75},
         "xA: Expected Assists": {"value": "0.27", "percentile": 83},
         "Key Passes": {"value": "1.48", "percentile": 62},
         "Passes into Final Third": {"value": "3.30", "percentile": 89},