Magazine_Optimization_Demo/Magazine_Optimization.py

import sys
sys.path.append('XGBoost_Prediction_Model/')

import warnings
warnings.filterwarnings("ignore")

import cv2 as cv
import numpy as np
from pulp import *
import Predict
import torch
from torchvision.io import read_image

#Global Paths for Models and Dictionaries
text_detection_model_path = 'XGBoost_Prediction_Model/EAST-Text-Detection/frozen_east_text_detection.pb'
LDA_model_pth = 'XGBoost_Prediction_Model/LDA_Model_trained/lda_model_best_tot.model'
training_ad_text_dictionary_path = 'XGBoost_Prediction_Model/LDA_Model_trained/object_word_dictionary'
training_lang_preposition_path = 'XGBoost_Prediction_Model/LDA_Model_trained/dutch_preposition'

def Preference_Matrix(Magazine_Pages, Magazine_Slots, Ad_Groups, Ad_Element_Sizes, 
                      Ad_embeddings, Ctpg_embeddings,
                      Textboxes=None, Obj_and_Topics=None, Costs=None,
                      Method='XGBoost'):
#Magazine_Pages: A list containing all paths to Magazine Ads and Editorials
#Magazine_Slots: 0 (right), 1 (left), 2 (full-page)
    
    #Costs Specification
    if Costs is None:
        Costs = np.ones(len(Magazine_Pages))

    #Separate Images into Ads and Counterpages
    Ads = []
    Counterpages = []
    Assign_ids = []
    Costs_Ctpg = []

    ad_locations = []
    prod_groups = []
    ad_elements = []
    ad_embeds = []
    ctpg_embeds = []

    if Textboxes is not None:
        ad_textbox = []; ctpg_textbox = []
    
    if Obj_and_Topics is not None:
        ad_num_obj = []; ctpg_num_obj = []
        ad_topic_weight = []; ctpg_topic_weight = []

    double_page_ad_attention = []
    double_page_brand_attention = []

    for i, path in enumerate(Magazine_Pages):
        if Magazine_Slots[i] == 2:
            if Textboxes is None:
                textboxes_curr = None
            else:
                textboxes_curr = Textboxes[i]

            if Obj_and_Topics is None:
                obj_and_topics_curr = None
            else:
                obj_and_topics_curr = Obj_and_Topics[i]

            if Method == 'XGBoost':
                ad_attention = Predict.Ad_Gaze_Prediction(input_ad_path=path, input_ctpg_path=None, ad_location=None,
                                                        text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, 
                                                        training_language='dutch', ad_embeddings=Ad_embeddings[i].reshape(1,768), ctpg_embeddings=Ctpg_embeddings[i].reshape(1,768),
                                                        surface_sizes=list(Ad_Element_Sizes[i]), Product_Group=list(Ad_Groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                                                        obj_detection_model_pth=None, num_topic=20, Gaze_Time_Type='Ad', Info_printing=False)

                brand_attention = Predict.Ad_Gaze_Prediction(input_ad_path=path, input_ctpg_path=None, ad_location=None,
                                                            text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                                                            training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, 
                                                            training_language='dutch', ad_embeddings=Ad_embeddings[i].reshape(1,768), ctpg_embeddings=Ctpg_embeddings[i].reshape(1,768), 
                                                            surface_sizes=list(Ad_Element_Sizes[i]), Product_Group=list(Ad_Groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                                                            obj_detection_model_pth=None, num_topic=20, Gaze_Time_Type='Brand', Info_printing=False)
                
            elif Method == 'CNN':
                img_curr = read_image(path)[:,89:921,:].unsqueeze(0)
                ad_img_CNN = img_curr[:,:,:,:640]
                ctpg_img_CNN = img_curr[:,:,:,640:]
                ad_location = torch.tensor([[1,1]])
                ad_attention = Predict.CNN_Prediction(ad_img_CNN, ctpg_img_CNN, ad_location, Gaze_Type='AG').item()
                brand_attention = Predict.CNN_Prediction(ad_img_CNN, ctpg_img_CNN, ad_location, Gaze_Type='BG').item()
            
            double_page_ad_attention.append(ad_attention/Costs[i])
            double_page_brand_attention.append(brand_attention/Costs[i])
        else:
            Assign_ids.append(i)
            img_curr = cv.imread(path)
            img_curr = cv.resize(img_curr, (1280,1024))
            _, w, _ = img_curr.shape
            page_width = w // 2
            ad_locations.append(1-Magazine_Slots[i])
            ctpg_location = Magazine_Slots[i]
            ad_img = img_curr[:, (Magazine_Slots[i]*page_width):((Magazine_Slots[i]+1)*page_width)]
            ctpg_img = img_curr[:, (ctpg_location*page_width):((ctpg_location+1)*page_width)]
            Ads.append(ad_img)
            Counterpages.append(ctpg_img)
            prod_groups.append(Ad_Groups[i])
            ad_elements.append(Ad_Element_Sizes[i])
            Costs_Ctpg.append(Costs[i])
            ad_embeds.append(Ad_embeddings[i])
            ctpg_embeds.append(Ctpg_embeddings[i])

            if Textboxes is not None:
                ad_textbox_curr, ctpg_textbox_curr = Textboxes[i]
                ad_textbox.append(ad_textbox_curr); ctpg_textbox.append(ctpg_textbox_curr)

            if Obj_and_Topics is not None:
                ad_obj_curr, ctpg_obj_curr, ad_topic_curr, ctpg_topic_curr = Obj_and_Topics[i]
                ad_num_obj.append(ad_obj_curr); ctpg_num_obj.append(ctpg_obj_curr)
                ad_topic_weight.append(ad_topic_curr); ctpg_topic_weight.append(ctpg_topic_curr)

    Ad_Attention_Preference = np.zeros((len(Ads),len(Counterpages)))
    Brand_Attention_Preference = np.zeros((len(Ads),len(Counterpages)))

    for i, ad in enumerate(Ads):
        print('Ad '+str(i)+" Assigning...")
        if Method == 'CNN':
            ad_images_stack = []
            ctpg_images_stack = []
            ad_locations_stack = []
        for j, ctpg in enumerate(Counterpages):
            
            # if ad_locations[j] == 0:
            #     new_image = np.concatenate((ad,ctpg),axis=1)
            # else:
            #     new_image = np.concatenate((ctpg,ad),axis=1)
            
            if Textboxes is not None:
                textboxes_curr = [ad_textbox[i],ctpg_textbox[j]]
            else:
                textboxes_curr = None

            if Obj_and_Topics is not None:
                obj_and_topics_curr = [ad_num_obj[i],ctpg_num_obj[j],ad_topic_weight[i],ctpg_topic_weight[j]]
            else:
                obj_and_topics_curr = None
            
            if Method == 'XGBoost':
                ad_attention = Predict.Ad_Gaze_Prediction(input_ad_path=ad, input_ctpg_path=ctpg, 
                                                        text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                                                        training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, 
                                                        training_language='dutch', ad_embeddings=ad_embeds[i].reshape(1,768), ctpg_embeddings=ctpg_embeds[i].reshape(1,768),
                                                        surface_sizes=list(ad_elements[i]), Product_Group=list(prod_groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                                                        obj_detection_model_pth=None, ad_location=ad_locations[j], num_topic=20, Gaze_Time_Type='Ad', Info_printing=False)
                brand_attention = Predict.Ad_Gaze_Prediction(input_ad_path=ad, input_ctpg_path=ctpg, 
                                                            text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                                                            training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, 
                                                            training_language='dutch', ad_embeddings=ad_embeds[i].reshape(1,768), ctpg_embeddings=ctpg_embeds[i].reshape(1,768),
                                                            surface_sizes=list(ad_elements[i]), Product_Group=list(prod_groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                                                            obj_detection_model_pth=None, ad_location=ad_locations[j], num_topic=20, Gaze_Time_Type='Brand', Info_printing=False)
                
                Ad_Attention_Preference[i,j] = ad_attention/Costs_Ctpg[j]
                Brand_Attention_Preference[i,j] = brand_attention/Costs_Ctpg[j]

            elif Method == 'CNN':
                ad_img_CNN = torch.tensor(ad).permute(2,0,1).unsqueeze(0)[:,:,89:921,:]
                ad_images_stack.append(ad_img_CNN)
                ctpg_img_CNN = torch.tensor(ctpg).permute(2,0,1).unsqueeze(0)[:,:,89:921,:]
                ctpg_images_stack.append(ctpg_img_CNN)
                ad_locations_stack.append(torch.tensor([[1,0]]))
                # ad_attention = Predict.CNN_Prediction(ad_img_CNN, ctpg_img_CNN, ad_location, Gaze_Type='AG').item()
                # brand_attention = Predict.CNN_Prediction(ad_img_CNN, ctpg_img_CNN, ad_location, Gaze_Type='BG').item()

        if Method == 'CNN':
            ad_images_stack = torch.cat(ad_images_stack,dim=0)
            ctpg_images_stack = torch.cat(ctpg_images_stack,dim=0)
            ad_locations_stack = torch.cat(ad_locations_stack,dim=0)
            ad_attentions = Predict.CNN_Prediction(ad_images_stack, ctpg_images_stack, ad_locations_stack, Gaze_Type='AG').to('cpu').squeeze()
            brand_attentions = Predict.CNN_Prediction(ad_images_stack, ctpg_images_stack, ad_locations_stack, Gaze_Type='BG').to('cpu').squeeze()
            Ad_Attention_Preference[i] = ad_attentions.numpy()/np.array(Costs_Ctpg)
            Brand_Attention_Preference[i] = brand_attentions.numpy()/np.array(Costs_Ctpg)


    return Ad_Attention_Preference, Brand_Attention_Preference, double_page_ad_attention, double_page_brand_attention, Assign_ids

def Preference_Matrix_different_magazine(Magzine_Target, Magzine_Ad, 
                                         Magazine_Slots_Target, Magazine_Slots_Ad, 
                                         Ad_Groups, Ad_Element_Sizes,
                                         Textboxes_Target=None, Textboxes_Ad=None,
                                         Obj_and_Topics_Target=None, Obj_and_Topics_Ad=None,
                                         Costs=None):
    #Separate Images into Ads and Counterpage
    Ads = []
    Counterpages = []
    Assign_ids_target = []
    Assign_ids_ad = []

    ad_locations = []
    prod_groups = []
    ad_elements = []

    if Textboxes_Target is not None:
        ad_textbox = []; ctpg_textbox = []
    
    if Obj_and_Topics_Target is not None:
        ad_num_obj = []; ctpg_num_obj = []
        ad_topic_weight = []; ctpg_topic_weight = []

    double_page_ad_attention = []
    double_page_brand_attention = []

    #Target magazine (Counterpage)
    for i, path in enumerate(Magzine_Target):
        if Magazine_Slots_Target[i] == 2:
            continue
        else:
            Assign_ids_target.append(i)
            img_curr = cv.imread(path)
            img_curr = cv.resize(img_curr, (1280,1024))
            _, w, _ = img_curr.shape
            page_width = w // 2
            ad_locations.append(Magazine_Slots_Target[i])
            ctpg_location = 1-Magazine_Slots_Target[i]
            ctpg_img = img_curr[:, (ctpg_location*page_width):((ctpg_location+1)*page_width)]
            Counterpages.append(ctpg_img)

            if Textboxes_Target is not None:
                _, ctpg_textbox_curr = Textboxes_Target[i]
                ctpg_textbox.append(ctpg_textbox_curr)

            if Obj_and_Topics_Target is not None:
                _, ctpg_obj_curr, _, ctpg_topic_curr = Obj_and_Topics_Target[i]
                ctpg_num_obj.append(ctpg_obj_curr)
                ctpg_topic_weight.append(ctpg_topic_curr)

    #Ad magazine (Ads)
    for i, path in enumerate(Magzine_Ad):
        if Magazine_Slots_Ad[i] == 2:
            if Textboxes_Ad is None:
                textboxes_curr = None
            else:
                textboxes_curr = Textboxes_Ad[i]

            if Obj_and_Topics_Ad is None:
                obj_and_topics_curr = None
            else:
                obj_and_topics_curr = Obj_and_Topics_Ad[i]

            ad_attention = Predict.Ad_Gaze_Prediction(input_ad_path=path, input_ctpg_path=None, text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                        training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, training_language='dutch', 
                        surface_sizes=list(Ad_Element_Sizes[i]), Product_Group=list(Ad_Groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                        obj_detection_model_pth=None, ad_location=None, num_topic=20, Gaze_Time_Type='Ad', Info_printing=False)
            
            brand_attention = Predict.Ad_Gaze_Prediction(input_ad_path=path, input_ctpg_path=None, text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                        training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, training_language='dutch', 
                        surface_sizes=list(Ad_Element_Sizes[i]), Product_Group=list(Ad_Groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                        obj_detection_model_pth=None, ad_location=None, num_topic=20, Gaze_Time_Type='Brand', Info_printing=False)
            double_page_ad_attention.append(ad_attention)
            double_page_brand_attention.append(brand_attention)
        else:
            Assign_ids_ad.append(i)
            img_curr = cv.imread(path)
            img_curr = cv.resize(img_curr, (1280,1024))
            _, w, _ = img_curr.shape
            page_width = w // 2
            ad_img = img_curr[:, (Magazine_Slots_Ad[i]*page_width):((Magazine_Slots_Ad[i]+1)*page_width)]
            Ads.append(ad_img)
            prod_groups.append(Ad_Groups[i])
            ad_elements.append(Ad_Element_Sizes[i])

            if Textboxes_Ad is not None:
                ad_textbox_curr, _ = Textboxes_Ad[i]
                ad_textbox.append(ad_textbox_curr)

            if Obj_and_Topics_Ad is not None:
                ad_obj_curr, _, ad_topic_curr, _ = Obj_and_Topics_Ad[i]
                ad_num_obj.append(ad_obj_curr)
                ad_topic_weight.append(ad_topic_curr)

    #Check costs on Ad position
    if Costs is None:
        Costs = np.ones(len(Counterpages))

    #Matrix
    if len(Ads) > len(Counterpages):
        return None
    else:
        Ad_Attention_Preference = np.zeros((len(Ads),len(Counterpages)))
        Brand_Attention_Preference = np.zeros((len(Ads),len(Counterpages)))
        for i, ad in enumerate(Ads):
            print('Ad '+str(i)+" Assigning...")
            for j, ctpg in enumerate(Counterpages):
                
                # if ad_locations[j] == 0:
                #     new_image = np.concatenate((ad,ctpg),axis=1)
                # else:
                #     new_image = np.concatenate((ctpg,ad),axis=1)
                
                if Textboxes_Target is not None:
                    textboxes_curr = [ad_textbox[i],ctpg_textbox[j]]
                else:
                    textboxes_curr = None

                if Obj_and_Topics_Target is not None:
                    obj_and_topics_curr = [ad_num_obj[i],ctpg_num_obj[j],ad_topic_weight[i],ctpg_topic_weight[j]]
                else:
                    obj_and_topics_curr = None
                
                ad_attention = Predict.Ad_Gaze_Prediction(input_ad_path=ad, input_ctpg_path=ctpg, text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                            training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, training_language='dutch', 
                            surface_sizes=list(ad_elements[i]), Product_Group=list(prod_groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                            obj_detection_model_pth=None, ad_location=ad_locations[j], num_topic=20, Gaze_Time_Type='Ad', Info_printing=False)
                brand_attention = Predict.Ad_Gaze_Prediction(input_ad_path=ad, input_ctpg_path=ctpg, text_detection_model_path=text_detection_model_path, LDA_model_pth=LDA_model_pth, 
                            training_ad_text_dictionary_path=training_ad_text_dictionary_path, training_lang_preposition_path=training_lang_preposition_path, training_language='dutch', 
                            surface_sizes=list(ad_elements[i]), Product_Group=list(prod_groups[i]), TextBoxes=textboxes_curr, Obj_and_Topics=obj_and_topics_curr,
                            obj_detection_model_pth=None, ad_location=ad_locations[j], num_topic=20, Gaze_Time_Type='Brand', Info_printing=False)
                Ad_Attention_Preference[i,j] = ad_attention/Costs[j]
                Brand_Attention_Preference[i,j] = brand_attention/Costs[j]
        return Ad_Attention_Preference, Brand_Attention_Preference, double_page_ad_attention, double_page_brand_attention, Assign_ids_ad, Assign_ids_target

def Assignment_Problem(costs, workers, jobs):
    #https://machinelearninggeek.com/solving-assignment-problem-using-linear-programming-in-python/

    prob = LpProblem("Assignment Problem", LpMinimize) 

    # The cost data is made into a dictionary
    costs= makeDict([workers, jobs], costs, 0)

    # Creates a list of tuples containing all the possible assignments
    assign = [(w, j) for w in workers for j in jobs]

    # A dictionary called 'Vars' is created to contain the referenced variables
    vars = LpVariable.dicts("Assign", (workers, jobs), 0, None, LpBinary)

    # The objective function is added to 'prob' first
    prob += (
        lpSum([vars[w][j] * costs[w][j] for (w, j) in assign]),
        "Sum_of_Assignment_Costs",
    )

    # There are row constraints. Each job can be assigned to only one employee.
    for j in jobs:
        prob+= lpSum(vars[w][j] for w in workers) == 1

    # There are column constraints. Each employee can be assigned to only one job.
    for w in workers:
        prob+= lpSum(vars[w][j] for j in jobs) == 1

    # The problem is solved using PuLP's choice of Solver
    prob.solve()

    return prob