# Title:    Load Data to WebAdMIT with the API
# Language: Python 3.7
# Author:   Greg Martin
# Date:     6/18/2019
# Contact:  gmartin@liaisonedu.com

# Notes on Loading Data to WebAdMIT with the API:
# Loading data to WebAdMIT is possible using the WebAdMIT API. Custom fields created in WebAdMIT can be accessed –
# data added, retrieved, and/or removed. Custom questions set up in the application can’t be accessed via the API;
# it is possible to retrieve custom question data using the Export Manager. Interactions with custom field data via
# the API are at the individual application level: in other words, each custom field is updated one at a time and not
# in a batch. That being the case, the best approach to loading data to WebAdMIT starts with a complete data set of
# all applications to be updated in WebAdMIT that includes the necessary information to identify the appropriate
# application and custom field, and the data to upload. This data set will be easiest to work with if there is one row
# per custom field to be updated. It is recommended to gather the necessary information from WebAdMIT – user identity,
# program ID, custom field ID, applicant CAS ID –  before constructing the data set for the upload

# Prepare Environment
# Make sure you have all the necessary libraries to interact with the API and access target directories
import requests  # make http requests, including API calls
from beautifultable import BeautifulTable   # present tables in output to easily review API responses
import time  # pause in between file generation status checks
import os   # find local directories
import pandas as pd     # tool for preparing data for upload to WebAdMIT

# WebAdMIT API Key
# First, you’ll need to get an API key from WebAdMIT.
# Log in to WebAdMIT and click on the “Account” link in the top righthand corner of the screen.
# Once there, click the “Edit My Account” button.
# About halfway down the next screen is a section called “API Key”.
# If you don’t already have an API key, click the “Generate New Key” button.
# If you do have a key, click “Show Key”.
# Record the alphanumeric string displayed here –
# you’ll need to include with every call you make to the WebAdMIT API.

apiKey = "1234567890abcdef1234567890abcdef"

# Root URLs
# The root URL for your API calls depends on your CAS.
# The root for all API calls is https://DOMAIN.webadmit.org.
# DOMAIN is your organization domain name.
# The default DOMAIN is api (https://api.webadmit.org).
# Contact Liaison customer service if you’re unsure of which root URL to use

rootUrl = "https://api.webadmit.org"


# Define a function for interacting with the WebAdMIT API
def WebAdMITAPI(rootUrl=rootUrl, endpoint="/api/v1/user_identities/", requestType="GET", apiKey=apiKey,
                requestBody={}, bodyJson=False):
    """
    General function to interact with the WebAdMIT API. This framework can accommodate most endpoints.
    User will need to parse response object for desired information.
    :param rootUrl: root URL for your CAS's API
    :param endpoint: API endpoint to call
    :param requestType: http request type to make
    :param apiKey: WebAdMIT API Key
    :param requestBody: http request payload in dictionary/JSON format
    :param bodyJson: Boolean param indicating whether the header of the http request should specify the body as JSON
    :return: JSON object from http response
    """
    requestBody = str(requestBody).replace("'", '"').replace('"true"', 'true').replace('"false"', 'false')
    if bodyJson:
        requestHeaders = {
            'x-api-key': apiKey
            , 'Content-Type': 'application/json'
        }
    else:
        requestHeaders = {
            'x-api-key': apiKey
        }
    responseJson = requests.request(requestType, rootUrl+endpoint, headers=requestHeaders, data=requestBody).json()
    return responseJson


# Define a function to express JSON strings as tables
def Tabular(listOfDicts):
    """
    Present lengthy JSON string in tabular format for ease of review.
    :param listOfDicts: JSON string from http response; should be a list of dictionaries
    :return: tabular format of JSON string
    """
    try:
        table = BeautifulTable()
        table.column_headers = list(listOfDicts[0].keys())
        end = len(listOfDicts)
        for i in range(0, end):
            table.insert_row(i, listOfDicts[i].values())
        return table
    except:
        listOfDicts = [listOfDicts]
        table = BeautifulTable()
        table.column_headers = list(listOfDicts[0].keys())
        end = len(listOfDicts)
        for i in range(0, end):
            table.insert_row(i, listOfDicts[i].values())
        return table


# Step 1: Select the User Identity
# Pull down all user identities accessible to the user with the “User Identity List” endpoint
# (GET /api/v1/user_identities
# https://liaison-intl.github.io/user_identity.html).
# Each user identity corresponds to a specific combination of cycle, association, and organization.
# Users frequently have more than one user identity, especially when there’s more than one CAS on campus.
# For example, a user will have one user identity for accessing CASPA 2018-2019 data,
# and a different one for accessing PTCAS 2018-2019 data.
# Think about which applicants you’re interested in and select the appropriate user identity –
# you’ll use the “id” for this user identity to retrieve the data.

# Construct the url to access the "User Identity List" endpoint
userIdentityEndpoint = "/api/v1/user_identities/"
# Make the request and store the response
userIdentities = WebAdMITAPI(rootUrl=rootUrl, endpoint="/api/v1/user_identities/", apiKey=apiKey)["user_identities"]
# Present the response in a table for review
print(Tabular(userIdentities))
# Select the user identity to be used going forward
userIdentityId = 123456789  # enter the selected user identity ID here

# Step 2: Select the Program
# Since custom fields in WebAdMIT can be defined at the program level, it’s necessary to specify the program when
# uploading data to an applicant’s record in WebAdMIT. Pull down a list of programs accessible to the user identity
# with the “Program List” endpoint
# (GET /api/v1/user_identities/:user_identity_id/programs
# https://liaison-intl.github.io/program.html).
# Identify the program to which you want to upload data. Use the program’s “name” and “organization_name” to identify
# the intended program. You’ll use the program’s “id” from the response to this call to upload data.

# Construct the url to access the “Program List” endpoint
programListEndpoint = "/api/v1/user_identities/" + str(userIdentityId) + "/programs"
# Make the request and store the response
programs = WebAdMITAPI(rootUrl=rootUrl, endpoint=programListEndpoint, apiKey=apiKey)["programs"]
# Present the response in a table for review
print(Tabular(programs))
# Select the program for which custom field data will be loaded
programId = 123456789   # enter the selected program ID

# Step 3: Select the Custom Field
# It’s possible to create multiple custom fields in WebAdMIT. You’ll need to identify the
# custom field that you want to upload data to. Pull down a list of custom fields present in
# the selected program with the “Custom Field List” endpoint
# (GET /api/v1/user_identities/:user_identity_id/programs/:program_id/custom_fields
# https://liaison-intl.github.io/custom_field.html).
# Use the “label” and “field_type” values to identify the custom field of interest.
# You’ll use the custom field “id” from this response when uploading data.

# Construct the url to access the “Custom Field List” endpoint
customFieldsEndpoint = "/api/v1/user_identities/" + str(userIdentityId) + "/programs/" + str(programId) + \
                          "/custom_fields"
# Make the request and store the response
customFields = WebAdMITAPI(rootUrl=rootUrl, endpoint=customFieldsEndpoint, apiKey=apiKey)["custom_fields"]
# Present the response in a table for review
print(Tabular(customFields))
# Store info on the custom fields of interest for later use
customFieldIds = [customField for customField in customFields if customField["id"] in
                  (123456, 123457, 123458, 123459)]  # enter the custom field IDs of interest here

# Step 4: Gather Data to be Loaded to WebAdMIT from Local Systems
# The user identity, program, and custom field information gathered above are necessary for each API call to load data
# to WebAdMIT. Two other pieces of information are also needed: the CAS ID for each applicant to be updated and the
# value to be loaded to the custom field. These two pieces of information will come from your local systems. It is
# recommended that CAS ID be included in all imports into your local systems.
# Without the CAS ID, it isn’t possible to upload data to custom fields in WebAdMIT.
#
# The value to be loaded to the custom field must match the custom field’s data type. There are 5 possible data types
# for WebAdMIT custom fields: Boolean, number, date, string, select. A Boolean field accepts Boolean values: true or
# false (without quotes). A number field accepts numeric values: e.g. 11 or 3400 (without quotes). A date field accepts
# date values in a specific format: “YYYY-MM-dd” e.g. “2016-01-31” (with quotes; must prefix single-digit months/days
# with “0”). A string field accepts any text value: e.g. “first generation”, “"000-0-0768" (with quotes). A select
# field accepts one of a set of predefined options determined when the custom field was created: e.g. “Option A” (with
# quotes; must match one of the predefined options). The “Custom Field List” endpoint (see above) will return the
# possible options for each custom field of select type.
#
# There are many ways to gather local data for upload to WebAdMIT. You might be able to query the local system’s
# database directly, pulling the data into an object in your coding language of choice that can be iterated through to
# upload each custom field value for each applicant. You might have to set up a flat file export from your local
# system, then read the data from that file into an object before loading.
#
# FOR THE PURPOSES OF THIS SAMPLE SCRIPT, WE'LL READ DATA FROM A FLAT FILE
# SAMPLE LOCAL DATA FILE INCLUDED: WebAdMIT_API-Load_Data_to_WebAdMIT-Custom_Field_Data.csv
# THIS SAMPLE DATA INCLUDES 5 DISTINCT CUSTOM FIELD VALUES ACROSS 5 COLUMNS FOR A NUMBER OF APPLICANTS (1 ROW PER APP)
# THE SCRIPT WILL READ THE DATA AND LOAD THE CUSTOM FIELD VALUES FOR ALL FIELDS FOR ALL APPLICANTS

# Gather local data from a flat file
currDir = os.getcwd()
localData = pd.read_csv(currDir + "/WebAdMIT_API-Load_Data_to_WebAdMIT-Custom_Field_Data.csv")

# Step 5: Load the Data to WebAdMIT Custom Fields
# Combining the list of applicants to be updated that includes each applicant’s CAS ID and intended custom field value
# with the user identity, program, and custom field information gathered above, you’re ready to load data to WebAdMIT.
# For each applicant, make a call to the “Custom Field Answer Store” endpoint
# (PUT /api/v1/user_identities/:user_identity_id/programs/:program_id/applicants_by_cas_id/:applicant_cas_id/custom_field_answers/:custom_field_id
# https://liaison-intl.github.io/custom_field_answer.html)
# including in the body of the request a JSON string matching the following format. Note that you’ll have to set the
# “field_type” to match the data type of the custom field you’re trying to update (see above). The “value” is the
# intended custom field value for the applicant.
# {
#     "custom_field_answer": {
#         "field_type": "select",
#         "value": "Option A"
#     }
# }

# Loop through each applicant in the local data file
review = []
for index, applicant in localData.iterrows():
    print("Applicant #" + str(index + 1) + ":\t\t" + str(applicant["cas_id"]) + "\t\t" + applicant["first_name"] + " " +
          applicant["last_name"])
    # Loop through each custom field column
    for field in [field for field in applicant.keys() if "custom_field" in field]:
        reviewApplicantField = {}
        reviewApplicantField["row#"] = index + 1
        reviewApplicantField["cas_id"] = applicant["cas_id"]
        reviewApplicantField["first_name"] = applicant["first_name"]
        reviewApplicantField["last_name"] = applicant["last_name"]
        fieldId = [customField["id"] for customField in customFieldIds if
                   customField["label"].lower() in field.replace("_", " ")][0]
        fieldType = [customField["field_type"] for customField in customFieldIds if
                   customField["label"].lower() in field.replace("_", " ")][0]
        print(str(fieldId) + "\t\t" + fieldType + "\t\t" + field + ":\t\t" + str(applicant[field]))
        # Construct the url to access the “Custom Field Answer Store” endpoint
        fieldLoadEndpoint = "/api/v1/user_identities/" + str(userIdentityId) + "/programs/" + str(programId) \
                            + "/applicants_by_cas_id/" + str(applicant["cas_id"]) \
                            + "/custom_field_answers/" + str(fieldId)
        # Construct the payload of the http request
        fieldValue = str(applicant[field]).lower() if isinstance(applicant[field], bool) else applicant[field]  # booleans must be lowercase
        requestBody = {
            "custom_field_answer": {
                "field_type": fieldType
                , "value": fieldValue
            }
        }
        # Make the request and store the response
        fieldLoadResponse = WebAdMITAPI(rootUrl=rootUrl, endpoint=fieldLoadEndpoint, requestType="PUT", apiKey=apiKey,
                                        requestBody=requestBody, bodyJson=True)["custom_field_answer"]
        reviewApplicantField[field + "_id"] = fieldId
        reviewApplicantField[field + "_type"] = fieldType
        reviewApplicantField[field + "value_to_load"] = applicant[field]
        reviewApplicantField[field + "_value_loaded"] = fieldLoadResponse["value"]
        reviewApplicantField[field + "_value_match"] = True if fieldLoadResponse["value"] == applicant[field] else False
        review.append(reviewApplicantField)

# Step 5: Quality Control on the Upload
# The response to the upload call will contain information confirming whether it was successful. Successful requests
# will return the ID, label, and data type of the custom field updated, along with the value loaded. To be extra
# careful, you might check the value returned in the response against the intended value you uploaded to make sure it
# loaded correctly.

# Review the values to be loaded against the values actually loaded
print(Tabular(review))