{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "syA7n33Fex63"
      },
      "source": [
        "# Recommender systems: introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h6mxo6veex65"
      },
      "source": [
        "# Similarités"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install wget"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "304_Fk6tfChX",
        "outputId": "b318e8ea-5b7b-4df9-f055-00d40e4fd5d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting wget\n",
            "  Downloading wget-3.2.zip (10 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9655 sha256=425e47be97363f1e8394d2fad91d55ff49522030a05fae2c4d5e7231e980081b\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/b3/0f/a40dbd1c6861731779f62cc4babcb234387e11d697df70ee97\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tNJqepU3ex65"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import wget\n",
        "import gzip\n",
        "import math\n",
        "import random\n",
        "from collections import defaultdict\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_AVkBkaex65"
      },
      "source": [
        "Téléchargement des données (avis sur des instruments de musique sur Amazon)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wvprFj_Vex65",
        "outputId": "5f361587-e062-4ca7-c417-a39ea8853ffb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Done!\n"
          ]
        }
      ],
      "source": [
        "filenames = ['amazon_reviews_us_Musical_Instruments_v1_00.tsv.gz']\n",
        "\n",
        "dataDir = './data'\n",
        "url = 'http://jmcauley.ucsd.edu/pml_data'\n",
        "\n",
        "if not os.path.exists(dataDir):\n",
        "    os.makedirs(dataDir)\n",
        "for filename in filenames:\n",
        "    wget.download(os.path.join(url, filename), out=dataDir)\n",
        "print(\"Done!\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ssllX7uVex66"
      },
      "outputs": [],
      "source": [
        "path = os.path.join(\n",
        "    dataDir, \"amazon_reviews_us_Musical_Instruments_v1_00.tsv.gz\")\n",
        "f = gzip.open(path, 'rt', encoding=\"utf8\")\n",
        "\n",
        "header = f.readline()\n",
        "header = header.strip().split('\\t')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IJ7TiTlsex67"
      },
      "source": [
        "Regardons les champs dans les données."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y8kmKZA5ex67",
        "outputId": "df9957c4-2627-4804-bfe4-6e311a1dd175"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['marketplace',\n",
              " 'customer_id',\n",
              " 'review_id',\n",
              " 'product_id',\n",
              " 'product_parent',\n",
              " 'product_title',\n",
              " 'product_category',\n",
              " 'star_rating',\n",
              " 'helpful_votes',\n",
              " 'total_votes',\n",
              " 'vine',\n",
              " 'verified_purchase',\n",
              " 'review_headline',\n",
              " 'review_body',\n",
              " 'review_date']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "header\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qDdU5hhCex68"
      },
      "source": [
        "On \"parse\" les données, convertissant au passage les entiers si nécessaire."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sJzOyHusex69"
      },
      "outputs": [],
      "source": [
        "dataset = []\n",
        "\n",
        "for line in f:\n",
        "    fields = line.strip().split('\\t')\n",
        "    d = dict(zip(header, fields))\n",
        "    d['star_rating'] = int(d['star_rating'])\n",
        "    d['helpful_votes'] = int(d['helpful_votes'])\n",
        "    d['total_votes'] = int(d['total_votes'])\n",
        "    dataset.append(d)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgZWuFJBex69"
      },
      "source": [
        "Regardons une ligne du dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yaPbRFY1ex69",
        "outputId": "5ef47037-ff0a-48fc-998d-775bde7cc308"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'marketplace': 'US',\n",
              " 'customer_id': '45610553',\n",
              " 'review_id': 'RMDCHWD0Y5OZ9',\n",
              " 'product_id': 'B00HH62VB6',\n",
              " 'product_parent': '618218723',\n",
              " 'product_title': 'AGPtek® 10 Isolated Output 9V 12V 18V Guitar Pedal Board Power Supply Effect Pedals with Isolated Short Cricuit / Overcurrent Protection',\n",
              " 'product_category': 'Musical Instruments',\n",
              " 'star_rating': 3,\n",
              " 'helpful_votes': 0,\n",
              " 'total_votes': 1,\n",
              " 'vine': 'N',\n",
              " 'verified_purchase': 'N',\n",
              " 'review_headline': 'Three Stars',\n",
              " 'review_body': 'Works very good, but induces ALOT of noise.',\n",
              " 'review_date': '2015-08-31'}"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "dataset[0]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oLFp7iehex6-"
      },
      "source": [
        "Créons quelques structures de données utiles pour la suite."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BCoBW1_Sex6-"
      },
      "outputs": [],
      "source": [
        "usersPerItem = defaultdict(set)  # Maps an item to the users who rated it\n",
        "itemsPerUser = defaultdict(set)  # Maps a user to the items that they rated\n",
        "itemNames = {}\n",
        "ratingDict = {}  # To retrieve a rating for a specific user/item pair\n",
        "\n",
        "for d in dataset:\n",
        "    user, item = d['customer_id'], d['product_id']\n",
        "    usersPerItem[item].add(user)\n",
        "    itemsPerUser[user].add(item)\n",
        "    ratingDict[(user, item)] = d['star_rating']\n",
        "    itemNames[item] = d['product_title']\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVvs3DCEex6-"
      },
      "source": [
        "Extrayons les moyennes par utilisateur et par élément (utile plus tard pour la prédiction de notes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jjvgUXZrex6-"
      },
      "outputs": [],
      "source": [
        "userAverages = {}\n",
        "itemAverages = {}\n",
        "\n",
        "for u in itemsPerUser:\n",
        "    rs = [ratingDict[(u, i)] for i in itemsPerUser[u]]\n",
        "    userAverages[u] = sum(rs) / len(rs)\n",
        "\n",
        "for i in usersPerItem:\n",
        "    rs = [ratingDict[(u, i)] for u in usersPerItem[i]]\n",
        "    itemAverages[i] = sum(rs) / len(rs)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZdrlhMzex6-"
      },
      "source": [
        "## Indicateurs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0zk9z6Oex6-"
      },
      "source": [
        "### Jaccard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cs1DWvZ2ex6-"
      },
      "outputs": [],
      "source": [
        "def Jaccard(s1, s2):\n",
        "    numer = len(s1.intersection(s2))\n",
        "    denom = len(s1.union(s2))\n",
        "    if denom == 0:\n",
        "        return 0\n",
        "    return numer / denom\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A7LRjhQcex6_"
      },
      "source": [
        "### Cosinus"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PhVvBWXnex6_"
      },
      "source": [
        "Implementation rapide pour des données en \"sets\"."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IekGQK4_ex6_"
      },
      "outputs": [],
      "source": [
        "def CosineSet(s1, s2):\n",
        "    # Not a proper implementation, operates on sets so correct for interactions only\n",
        "    numer = len(s1.intersection(s2))\n",
        "    denom = math.sqrt(len(s1)) * math.sqrt(len(s2))\n",
        "    if denom == 0:\n",
        "        return 0\n",
        "    return numer / denom\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EnULCTxwex6_"
      },
      "source": [
        "autre implémentation (avec les notes cette fois)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E1OEA20gex7A"
      },
      "outputs": [],
      "source": [
        "def Cosine(i1, i2):\n",
        "    # Between two items\n",
        "    inter = usersPerItem[i1].intersection(usersPerItem[i2])\n",
        "    numer = 0\n",
        "    denom1 = 0\n",
        "    denom2 = 0\n",
        "    for u in inter:\n",
        "        numer += ratingDict[(u, i1)]*ratingDict[(u, i2)]\n",
        "    for u in usersPerItem[i1]:\n",
        "        denom1 += ratingDict[(u, i1)]**2\n",
        "    for u in usersPerItem[i2]:\n",
        "        denom2 += ratingDict[(u, i2)]**2\n",
        "    denom = math.sqrt(denom1) * math.sqrt(denom2)\n",
        "    if denom == 0:\n",
        "        return 0\n",
        "    return numer / denom\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-n8Or6Xex7A"
      },
      "source": [
        "### Pearson"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "okkyBtszex7A"
      },
      "outputs": [],
      "source": [
        "def Pearson(i1, i2):\n",
        "    # Between two items\n",
        "    iBar1 = itemAverages[i1]\n",
        "    iBar2 = itemAverages[i2]\n",
        "    inter = usersPerItem[i1].intersection(usersPerItem[i2])\n",
        "    numer = 0\n",
        "    denom1 = 0\n",
        "    denom2 = 0\n",
        "    for u in inter:\n",
        "        numer += (ratingDict[(u, i1)] - iBar1)*(ratingDict[(u, i2)] - iBar2)\n",
        "    for u in inter:  # usersPerItem[i1]:\n",
        "        denom1 += (ratingDict[(u, i1)] - iBar1)**2\n",
        "    # for u in usersPerItem[i2]:\n",
        "        denom2 += (ratingDict[(u, i2)] - iBar2)**2\n",
        "    denom = math.sqrt(denom1) * math.sqrt(denom2)\n",
        "    if denom == 0:\n",
        "        return 0\n",
        "    return numer / denom\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pc2SKmx8ex7A"
      },
      "source": [
        "## Requêtes et similarité"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sgfH801Jex7B"
      },
      "source": [
        "On recherche les plus similaires, en fonction d'une similarité (Jaccard)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_AFHVWBaex7B"
      },
      "outputs": [],
      "source": [
        "def mostSimilar(i, N):\n",
        "    similarities = []\n",
        "    users = usersPerItem[i]\n",
        "    for i2 in usersPerItem:\n",
        "        if i2 == i:\n",
        "            continue\n",
        "        sim = Jaccard(users, usersPerItem[i2])\n",
        "        # sim = Pearson(i, i2) # Could use alternate similarity metrics straightforwardly\n",
        "        similarities.append((sim, i2))\n",
        "    similarities.sort(reverse=True)\n",
        "    return similarities[:10]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7jOanzg0ex7B"
      },
      "source": [
        "Prenons un élément comme \"requête\"."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qKLDjYtzex7B",
        "outputId": "407747ed-a357-48db-e456-af3a2c3bda73"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'marketplace': 'US',\n",
              " 'customer_id': '6111003',\n",
              " 'review_id': 'RIZR67JKUDBI0',\n",
              " 'product_id': 'B0006VMBHI',\n",
              " 'product_parent': '603261968',\n",
              " 'product_title': 'AudioQuest LP record clean brush',\n",
              " 'product_category': 'Musical Instruments',\n",
              " 'star_rating': 3,\n",
              " 'helpful_votes': 0,\n",
              " 'total_votes': 1,\n",
              " 'vine': 'N',\n",
              " 'verified_purchase': 'Y',\n",
              " 'review_headline': 'Three Stars',\n",
              " 'review_body': 'removes dust. does not clean',\n",
              " 'review_date': '2015-08-31'}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "dataset[2]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x8H_Izzrex7B"
      },
      "outputs": [],
      "source": [
        "query = dataset[2]['product_id']\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ciZ4Nh9Uex7B"
      },
      "source": [
        "Et recherchons les éléments les plus similaires."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PPNSrxfJex7C"
      },
      "outputs": [],
      "source": [
        "ms = mostSimilar(query, 10)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wH1qvm6hex7C",
        "outputId": "88f4d31f-74ce-4f13-9da4-d4c5279d3df3"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.028446389496717725, 'B00006I5SD'),\n",
              " (0.01694915254237288, 'B00006I5SB'),\n",
              " (0.015065913370998116, 'B000AJR482'),\n",
              " (0.014204545454545454, 'B00E7MVP3S'),\n",
              " (0.008955223880597015, 'B001255YL2'),\n",
              " (0.008849557522123894, 'B003EIRVO8'),\n",
              " (0.008333333333333333, 'B0015VEZ22'),\n",
              " (0.00821917808219178, 'B00006I5UH'),\n",
              " (0.008021390374331552, 'B00008BWM7'),\n",
              " (0.007656967840735069, 'B000H2BC4E')]"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "ms\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bDaxNB6Aex7C"
      },
      "source": [
        "Affichons les noms de l'élément requête et des recommandations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "fJIAxunyex7C",
        "outputId": "b0e0b547-e98d-491b-857b-5e40b2587260"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'AudioQuest LP record clean brush'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "itemNames[query]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b956pYEPex7C",
        "outputId": "3117c4d0-a53c-407e-fa78-721397f4a4c6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Shure SFG-2 Stylus Tracking Force Gauge',\n",
              " 'Shure M97xE High-Performance Magnetic Phono Cartridge',\n",
              " 'ART Pro Audio DJPRE II Phono Turntable Preamplifier',\n",
              " 'Signstek Blue LCD Backlight Digital Long-Playing LP Turntable Stylus Force Scale Gauge Tester',\n",
              " 'Audio Technica AT120E/T Standard Mount Phono Cartridge',\n",
              " 'Technics: 45 Adaptor for Technics 1200 (SFWE010)',\n",
              " 'GruvGlide GRUVGLIDE DJ Package',\n",
              " 'STANTON MAGNETICS Record Cleaner Kit',\n",
              " 'Shure M97xE High-Performance Magnetic Phono Cartridge',\n",
              " 'Behringer PP400 Ultra Compact Phono Preamplifier']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "[itemNames[x[1]] for x in ms]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jvTWh_dex7D"
      },
      "source": [
        "Implémentation plus rapide."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Czor8qGSex7D"
      },
      "outputs": [],
      "source": [
        "def mostSimilarFast(i, N):\n",
        "    similarities = []\n",
        "    users = usersPerItem[i]\n",
        "    candidateItems = set()\n",
        "    for u in users:\n",
        "        candidateItems = candidateItems.union(itemsPerUser[u])\n",
        "    for i2 in candidateItems:\n",
        "        if i2 == i:\n",
        "            continue\n",
        "        sim = Jaccard(users, usersPerItem[i2])\n",
        "        similarities.append((sim, i2))\n",
        "    similarities.sort(reverse=True)\n",
        "    return similarities[:N]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0kEyJHrIex7D"
      },
      "source": [
        "Vérifions que les éléments recoommandés sont les mêmes."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-iNBeQ9Sex7D",
        "outputId": "4ed78e4a-4e11-4141-e604-2266729bd6ad"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.028446389496717725, 'B00006I5SD'),\n",
              " (0.01694915254237288, 'B00006I5SB'),\n",
              " (0.015065913370998116, 'B000AJR482'),\n",
              " (0.014204545454545454, 'B00E7MVP3S'),\n",
              " (0.008955223880597015, 'B001255YL2'),\n",
              " (0.008849557522123894, 'B003EIRVO8'),\n",
              " (0.008333333333333333, 'B0015VEZ22'),\n",
              " (0.00821917808219178, 'B00006I5UH'),\n",
              " (0.008021390374331552, 'B00008BWM7'),\n",
              " (0.007656967840735069, 'B000H2BC4E')]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "mostSimilarFast(query, 10)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nCnl1Cv9ex7E"
      },
      "source": [
        "## Notes basées sur des similarités"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ijLx06Ctex7E"
      },
      "source": [
        "On va maintenant utiliser nos fonctions de similarité pour estimer les notes. On commence par quelques structures de données utiles pour la suite."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-NNW6GK7ex7E"
      },
      "outputs": [],
      "source": [
        "reviewsPerUser = defaultdict(list)\n",
        "reviewsPerItem = defaultdict(list)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tThdtuFRex7E"
      },
      "outputs": [],
      "source": [
        "for d in dataset:\n",
        "    user, item = d['customer_id'], d['product_id']\n",
        "    reviewsPerUser[user].append(d)\n",
        "    reviewsPerItem[item].append(d)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0zEBaUYsex7E"
      },
      "outputs": [],
      "source": [
        "ratingMean = sum([d['star_rating'] for d in dataset]) / len(dataset)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f-emH1wUex7E",
        "outputId": "43ed078a-1054-4ae0-e1ea-13469175634f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4.251102772543146"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "ratingMean\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sgSs_8AHex7F"
      },
      "source": [
        "Un exemple d'heuristique de recommandation (d'autres sont évidemment possibles)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jmtttoriex7F"
      },
      "outputs": [],
      "source": [
        "def predictRating(user, item):\n",
        "    ratings = []\n",
        "    similarities = []\n",
        "    for d in reviewsPerUser[user]:\n",
        "        i2 = d['product_id']\n",
        "        if i2 == item:\n",
        "            continue\n",
        "        ratings.append(d['star_rating'] - itemAverages[i2])\n",
        "        similarities.append(Jaccard(usersPerItem[item], usersPerItem[i2]))\n",
        "    if (sum(similarities) > 0):\n",
        "        weightedRatings = [(x*y) for x, y in zip(ratings, similarities)]\n",
        "        return itemAverages[item] + sum(weightedRatings) / sum(similarities)\n",
        "    else:\n",
        "        # User hasn't rated any similar items\n",
        "        return ratingMean\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o9jEGpW7ex7F",
        "outputId": "6d59836e-0ea4-4037-b458-84794e745cb8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'marketplace': 'US',\n",
              " 'customer_id': '14640079',\n",
              " 'review_id': 'RZSL0BALIYUNU',\n",
              " 'product_id': 'B003LRN53I',\n",
              " 'product_parent': '986692292',\n",
              " 'product_title': 'Sennheiser HD203 Closed-Back DJ Headphones',\n",
              " 'product_category': 'Musical Instruments',\n",
              " 'star_rating': 5,\n",
              " 'helpful_votes': 0,\n",
              " 'total_votes': 0,\n",
              " 'vine': 'N',\n",
              " 'verified_purchase': 'Y',\n",
              " 'review_headline': 'Five Stars',\n",
              " 'review_body': 'Nice headphones at a reasonable price.',\n",
              " 'review_date': '2015-08-31'}"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "dataset[1]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpMEfPgKex7F"
      },
      "source": [
        "Prédisons une note pour un ensemble user/item."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WbCXeXKCex7G"
      },
      "outputs": [],
      "source": [
        "u, i = dataset[1]['customer_id'], dataset[1]['product_id']\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3kqw9fN3ex7G",
        "outputId": "a065260c-26a2-4478-a7e0-1be243803404"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4.509357030989021"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "predictRating(u, i)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vY5Kihi1ex7G"
      },
      "source": [
        "Calculons la MSE pour ce modèle."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JgWFXlKtex7G"
      },
      "outputs": [],
      "source": [
        "def MSE(predictions, labels):\n",
        "    differences = [(x-y)**2 for x, y in zip(predictions, labels)]\n",
        "    return sum(differences) / len(differences)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gS_yApRXex7H"
      },
      "source": [
        "Comparons à un prédicteur trivial qui prédirait toujours la moyenne."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zYQkVG78ex7H"
      },
      "outputs": [],
      "source": [
        "alwaysPredictMean = [ratingMean for d in dataset]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pNd2jUrQex7H"
      },
      "source": [
        "Calculons les prédictions pour tous les éléments (attention : lent !)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i0_wq9fyex7H"
      },
      "outputs": [],
      "source": [
        "simPredictions = [predictRating(\n",
        "    d['customer_id'], d['product_id']) for d in dataset]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0haJWpayex7H"
      },
      "outputs": [],
      "source": [
        "labels = [d['star_rating'] for d in dataset]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zLRhWvv2ex7I",
        "outputId": "02a34840-b26b-42ed-de9f-068132d07eb1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.4796142779564334"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ],
      "source": [
        "MSE(alwaysPredictMean, labels)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GxJeGqQ5ex7I",
        "outputId": "b77f245a-7cab-4702-9d7b-895a08a2c3a2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.44672577948388"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ],
      "source": [
        "MSE(simPredictions, labels)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oe0wMMSEex7I"
      },
      "source": [
        "## Exercices"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O-ZA5gRrex7J"
      },
      "source": [
        "### 1"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Exercice : construisons une mesure de qualité simple pour un système de recommandation reposant sur la similarité.\n",
        "\n",
        "Un système de recommandation d'article à article peut être considéré comme « utile » s'il a tendance à classer les articles i et j qui ont tous deux été achetés par u comme étant plus similaires que deux articles qui n'ont pas été achetés par le même utilisateur.\n",
        "\n",
        "Pour chaque utilisateur, prendre au hasard deux de ses interactions i et j, ainsi qu'une troisième interaction k ∈ I non achetée par u. Mesurer la fréquence à laquelle le système note Sim(i, j) ≥ Sim(i, k). Calculer cette mesure pour les similitudes de Jaccard, Cosinus et de Pearson (ou d'autres variantes) pour déterminer laquelle est la mieux adaptée à cet ensemble de données."
      ],
      "metadata": {
        "id": "BFBBUllhk7zH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cx6ec9P8ex7J"
      },
      "outputs": [],
      "source": [
        "def simTest(simFunction, nUserSamples):\n",
        "    sims = []\n",
        "    randomSims = []\n",
        "\n",
        "    items = set(usersPerItem.keys())\n",
        "    users = list(itemsPerUser.keys())\n",
        "\n",
        "    for u in random.sample(users, nUserSamples):\n",
        "        itemsU = set(itemsPerUser[u])\n",
        "        if len(itemsU) < 2:\n",
        "            continue  # User needs at least two interactions\n",
        "        (i, j) = random.sample(list(itemsU), 2)\n",
        "        k = random.sample(list(items.difference(itemsU)), 1)[0] # Also convert this set to a list\n",
        "        usersi = usersPerItem[i].difference(set([u]))\n",
        "        usersj = usersPerItem[j].difference(set([u]))\n",
        "        usersk = usersPerItem[k].difference(set([u]))\n",
        "        sims.append(simFunction(usersi, usersj))\n",
        "        randomSims.append(simFunction(usersi, usersk))\n",
        "\n",
        "    print(\"Average similarity = \" + str(sum(sims)/len(sims)))\n",
        "    print(\"Average similarity (with random item) = \" +\n",
        "          str(sum(randomSims)/len(randomSims)))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l1gagBAeex7J",
        "outputId": "fa23183d-a24d-452a-8150-859780b4e62d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average similarity = 0.00229373971692631\n",
            "Average similarity (with random item) = 0.0\n"
          ]
        }
      ],
      "source": [
        "simTest(Jaccard, 1000)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IT3osccNex7J",
        "outputId": "cd5a5043-0be2-4f65-aeda-6ff0f21c837b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average similarity = 0.005178104536456735\n",
            "Average similarity (with random item) = 0.0001745657773678741\n"
          ]
        }
      ],
      "source": [
        "simTest(CosineSet, 1000)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0gR05iPZex7K"
      },
      "source": [
        "### Avec historique"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Nous n'avons pour le moment pas produit de recommandations basées sur l'historique de l'utilisateur. Cependant, il est possible de le faire de plusieurs manières, par exemple :\n",
        "\n",
        "- recommander un article i sur la base de la similarité moyenne avec tous les\n",
        "  articles j de l'historique de l'utilisateur ;\n",
        "- au lieu de faire la moyenne de tous les articles de l'historique de\n",
        "  l'utilisateur, faire la moyenne des K derniers articles seulement, ou pondérer\n",
        "la moyenne par la \"récence\" ;\n",
        "- sélectionner un article consommé par un utilisateur très similaire ;\n",
        "- etc.\n",
        "\n",
        "Explorer des solutions telles que celles mentionnées ci-dessus afin de déterminer celle qui est la plus à même de recommander les interactions futures des utilisateurs sur la base de leur historique.\n",
        "\n",
        "Pour les besoins de l'évaluation, il est utile que les variantes associent un score r(u, i) à chaque recommandation candidate ; par exemple, le score pourrait être la moyenne de la similarité cosinus entre i et les éléments j dans l'historique de u ; ou le score pourrait être la similarité de Jaccard entre u et l'utilisateur v le plus similaire qui a consommé i. Les méthodes peuvent ensuite être comparées en utilisant une approche similaire à l'exercice 1 : c'est-à-dire, la méthode a-t-elle tendance à attribuer des scores plus élevés aux éléments (retenus) avec lesquels l'utilisateur a interagi par rapport à des éléments choisis au hasard."
      ],
      "metadata": {
        "id": "XLsuUx0BlETY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CyIbNywDex7K"
      },
      "outputs": [],
      "source": [
        "items = set(usersPerItem.keys())\n",
        "users = set(itemsPerUser.keys())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7zpBFmyAex7K"
      },
      "outputs": [],
      "source": [
        "# 1: Average cosine similarity between i and items in u's history\n",
        "def rec1score(u, i, userHistory):\n",
        "    if len(userHistory) == 0:\n",
        "        return 0\n",
        "    averageSim = []\n",
        "    s1 = usersPerItem[i].difference(set([u]))\n",
        "    for h in userHistory:\n",
        "        s2 = usersPerItem[h].difference(set([u]))\n",
        "        averageSim.append(Jaccard(s1, s2))\n",
        "    averageSim = sum(averageSim)/len(averageSim)\n",
        "    return averageSim\n",
        "\n",
        "# 2: Jaccard similarity with most similar user who has consumed i\n",
        "\n",
        "\n",
        "def rec2score(u, i, userHistory):\n",
        "    bestSim = None\n",
        "    for v in usersPerItem[i]:\n",
        "        if u == v:\n",
        "            continue\n",
        "        sim = Jaccard(userHistory, itemsPerUser[v])\n",
        "        if bestSim == None or sim > bestSim:\n",
        "            bestSim = sim\n",
        "    if bestSim == None:\n",
        "        return 0\n",
        "    return bestSim\n",
        "\n",
        "# Generate a recommendation for a user based on a given scoring function\n",
        "\n",
        "\n",
        "def rec(u, score):\n",
        "    history = itemsPerUser[u]\n",
        "    if len(history) > 5:  # If the history is too long, just take a sample\n",
        "        history = random.sample(history, 5)\n",
        "    bestItem = None\n",
        "    bestScore = None\n",
        "    for i in items:\n",
        "        if i in itemsPerUser[u]:\n",
        "            continue\n",
        "        s = score(u, i, history)\n",
        "        if bestItem == None or s > bestScore:\n",
        "            bestItem = i\n",
        "            bestScore = s\n",
        "\n",
        "    return bestItem, bestScore\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zgg_K3arex7K"
      },
      "outputs": [],
      "source": [
        "u = random.sample(list(users), 1)[0]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JZ7BgOKmex7K",
        "outputId": "191ad8d6-89b9-4a5c-d6ff-66f06ec736c2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('B00N5J2M54', 1.0)"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ],
      "source": [
        "rec(u, rec1score)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9u7-w98Iex7L",
        "outputId": "8568ca51-c36d-43e3-e943-f9ad64a51a47"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('B00N5J2M54', 0.125)"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ],
      "source": [
        "rec(u, rec2score)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0KoGOlHVex7L"
      },
      "outputs": [],
      "source": [
        "def recTest(simFunction, nUserSamples):\n",
        "    items = list(set(usersPerItem.keys()))\n",
        "    users = list(itemsPerUser.keys())\n",
        "\n",
        "    better = 0\n",
        "    worse = 0\n",
        "\n",
        "    for u in random.sample(users, nUserSamples):\n",
        "        itemsU = set(itemsPerUser[u])\n",
        "        if len(itemsU) < 2:\n",
        "            continue\n",
        "        i = random.sample(list(itemsU), 1)[0]\n",
        "        uWithheld = itemsU.difference(set([i]))\n",
        "        j = random.sample(list(items), 1)[0]\n",
        "\n",
        "        si = simFunction(u, i, uWithheld)\n",
        "        sj = simFunction(u, j, uWithheld)\n",
        "\n",
        "        if si > sj:\n",
        "            better += 1\n",
        "        if sj > si:\n",
        "            worse += 1\n",
        "\n",
        "    print(\"Better than random \" + str(better) + \" times\")\n",
        "    print(\"Worse than random \" + str(worse) + \" times\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9xPJfRnex7L"
      },
      "source": [
        "Les résultats sur cet ensemble de données ne sont pas les plus intéressants. On pourrait essayer avec un ensemble de données plus dense (de sorte que de nombreux éléments aient une similarité non nulle) pour obtenir des résultats plus intéressants."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CDX0zcDEex7L",
        "outputId": "e7f1486c-c809-4ba8-9df2-41dc456e7dde"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Better than random 333 times\n",
            "Worse than random 2 times\n"
          ]
        }
      ],
      "source": [
        "recTest(rec1score, 5000)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MQzMD9MYex7L",
        "outputId": "5c1f332a-f02c-4863-dcc0-bb15e7617626"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Better than random 313 times\n",
            "Worse than random 6 times\n"
          ]
        }
      ],
      "source": [
        "recTest(rec2score, 5000)\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.13.3"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}