From 7400e1496fccc58b4bf0e3842f827127c1cbcae1 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 5 Dec 2023 18:54:28 +0530 Subject: [PATCH 01/91] files updated for generativeAI --- HelloWorld.java | 5 ++ __pycache__/api_json.cpython-310.pyc | Bin 0 -> 1825 bytes __pycache__/api_json1.cpython-310.pyc | Bin 0 -> 1893 bytes __pycache__/api_json2.cpython-310.pyc | Bin 0 -> 2443 bytes api_json.py | 69 +++++++++++++++++ api_json1.py | 71 +++++++++++++++++ chat-simple.py | 66 ++++++++++++++++ chat.py | 42 +++++++++++ err.py | 34 +++++++++ fetch_list.py | 84 +++++++++++++++++++++ fn_json_api.py | 94 +++++++++++++++++++++++ his1.py | 86 +++++++++++++++++++++ his2.py | 105 ++++++++++++++++++++++++++ his3.py | 80 ++++++++++++++++++++ his4.py | 83 ++++++++++++++++++++ his5.py | 17 +++-- his6.py | 103 +++++++++++++++++++++++++ history.py | 73 ++++++++++++++++++ input.txt | 1 + java_err.java | 37 +++++++++ json_schema.png | Bin 0 -> 50232 bytes list_op.py | 29 +++++++ main.py | 44 +++++++++++ main2.py | 44 +++++++++++ requirements.txt | 2 + test.py | 79 +++++++++++++++++++ test1.py | 92 ++++++++++++++++++++++ tt.txt | 62 +++++++++++++++ 28 files changed, 1395 insertions(+), 7 deletions(-) create mode 100644 HelloWorld.java create mode 100644 __pycache__/api_json.cpython-310.pyc create mode 100644 __pycache__/api_json1.cpython-310.pyc create mode 100644 __pycache__/api_json2.cpython-310.pyc create mode 100644 api_json.py create mode 100644 api_json1.py create mode 100644 chat-simple.py create mode 100644 chat.py create mode 100644 err.py create mode 100644 fetch_list.py create mode 100644 fn_json_api.py create mode 100644 his1.py create mode 100644 his2.py create mode 100644 his3.py create mode 100644 his4.py create mode 100644 his6.py create mode 100644 history.py create mode 100644 input.txt create mode 100644 java_err.java create mode 100644 json_schema.png create mode 100644 list_op.py create mode 100644 main.py create mode 100644 main2.py create mode 100644 requirements.txt create mode 100644 test.py create mode 100644 test1.py create mode 100644 tt.txt diff --git a/HelloWorld.java b/HelloWorld.java new file mode 100644 index 0000000..70fd330 --- /dev/null +++ b/HelloWorld.java @@ -0,0 +1,5 @@ +public class HelloWorld { + public static void main(String[] args) { + System.out.println("Hello, World!"); + } +} diff --git a/__pycache__/api_json.cpython-310.pyc b/__pycache__/api_json.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c22dfd1a829d9a81b732648e782657e2eef04fc4 GIT binary patch literal 1825 zcmZux&2Jnv6t_JeJNuPxnxttEh`AuKhkSO`FDlw9v``^Ml~M$?B8{fwx7kT{KCr!M zcD1>XUVG!np}oP41AmRLNJv~bBP0sXlTFn~wdH5~jpzCI-uw9#?RJA;`}OAxb^}6w zx0`pHhnssi)Nd#R5p+nno!O9PjI+$)PUdo#o_+JUCmir1U$`i1!b2GdA7vfbDdCNM z!kai+q9%eDgttXqG;p4S#;Z%B`I>?wTBo#6MH@POCg#MY7v3r5^Z5O;=)9(4UR-`b zP_qDwV&Raf^u5E2>_HN$n-HIU=y^UUyT#F&Ol_b4e3AIpWjp1C!v5iCZ zP)yoWM7)Br#o^#^SI7yS(r?Kwafv3H_Stxa65F3*ym!fS()D&sEg2MP0!muzrj~(H z@jjrs$Tj4;%S`Y!j2}o@NYi+f3plRURxd%7MXrFiG#u+__Qx+oHqq2So7=bdzEL36 zUKuA4rh4xSP=~rG_r8Q&#yUNM@cyH{xJ;u1Rpe{s*v#*Pj?S${*66%-uZ4oqMds5E z^(p&Lwo!2*8ZIg<6RT((;6jsER8Wiv`-#1#m`_Kzc73kfXPPWyd^9_74qV|)sqihU zkWqEyMB`;}!n8MKr}V_BY&ECOxh?I#qEp9OojSXu+uAYi2>rx}xUv|6tl|B}S7W6i zn|$`P7=^KfFwW82Fjgv6I?i>d2eA$ZFf3JALQxJO?5DX1)4Y%D!suW`xTi5MdcR+_ zKklZXL&$9R*WD^%-GKX$@25GKV3sDqy71}GxsO0l~3Ez(( zPsWjD=1`A$2U)041Lqp!Dm={VZvl5}W)U-dgbA(CMFVuCkYnS=C0Y^2KchLH+eXi} ziws?=UWT7Yh&5D6GadU6h_^{rUDq^h^rAE`N1C^0PE^U~4SXy!Gt#EFBG#F(oG|SK zuZo_hdJsJ;^a`rfrzkosa01HSFvtDZ_Zhv|{l9}&hc&26diA7T^5M8!o>Uc4FQx3=7GkaZDS&|mb`|)D1$|BA6bv&Uga|0Go z`2j8_bL;QE0F|KbLbInUI2O4kD$oc^~MbU!~acP1m%HlMS zqDg&o>&Ds!_RaL($JV%U2ZhqcEeol;%Z3$-*ZNpfu>wdNXNl~dUA}}X9@x^L$0eMt zp2Z4G)9Ne-ZOyIO%^{49gA|kt_U{V6Ee}<0Io^dgI!enbzQ#cZ-r~&T45G-?Mn`E< z$lN$`l)u;Q;7(SE(GWhjC9+~zKo)7he5^=SXRhPoJ_v%3uF}A3(7+GqB6Z*V4>$+W A`2YX_ literal 0 HcmV?d00001 diff --git a/__pycache__/api_json1.cpython-310.pyc b/__pycache__/api_json1.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3963d05e34fcdaa9e09d516335068e7470148a85 GIT binary patch literal 1893 zcmZuxOK%)S5bo}I?06r}I$0+%4>1RjSc$A3S&=v(lE@?wAz6_egfg-;8c)}H$KH9+ z-RpQ);|tC;H*N`1a^!Dt;>NwNNJv~bBMyY>S=)+Hx>YsRHQm+q_^R_(t4?qXf6L$( zkC5NZ;?03NatBHNii{IZM-invmwrJ;S1;SMBJ< zdAdBkxVEx({o2ZnYb&eQ3IPWRJPCZ*flvF_*H%{hH*faWBZzo^r60%FBZ$}IxW67# zDQ0Pl#OT`h$$aVI>>A09oXjgrKS5P=zNaekP)U zrv6#mxV8PQ1R=MJC)1$8?mm;_Ny3$vLt(;}*AHuw8m& zm-b7-ow8cLq7>7!$M!7lmbL+$0UJ)j8_&EWN7*I9=*U$Lo@tf{*%z!jNclq;71A#tFGk=Gl8pOFHb7Wm zTJo%i$+bm30%5|^_87AmI0|c|b_dn?<96a>_$RBsZdb+YdVvesZjyoaPRb~Q2KrI} zrW?^2)YIrNRQWz+67A(hSOg0L0pE`yizlH$=}1k2HX>4@22M5Fmbf{XJBy)P({5DY zey-gaxP$o(3~M7#F`lX)+*m-QpsJMaAT$mT2~~9A5EGG$iEf(@@UwbLG|NV-NNXmG znTigBdXg1mRUsj0o&+^p)v=LIE)%Bm73gNnz(u!WbSQk1sA0I1t0h#)50Q14XWOsa zj%zV`e!72q%{Hr3n~HaFVyobl%9s{Pi$dgR1;|^)AYch|D_feS6FTdF^GDh9G zaps%}LJ{dH%Bg9CXY2>cZAB^voz3J&?EdMc&D@{W#6RspiBN1HZq-;~e{%?24k}7U z;tCqn>8s$@Bxb23&S)@vM`*^t*+hZt>9cu?%*%e z1H)V@Z5O#v-7A{qGN=vk4aOHknCDMm2awt2izo%2c}mozfSI%tNze_WGte_{bj>XG zVWKSruefYZn=mSjRy7asF1*o_N>usP76x$UXBMRphPpOBNa9>%+7jdJtzmngr#T;w n;0yDl%8!wGq(eRC;saJqW?MF{J=j+)eK?GX-Tzx`J-bYjafZ&AFK8bD5`gFizHgIBR;KnYUe)D3l-s8tUcZAmqxCH~8{Pf|WPKGGBSlSnu4utMqsO?#;3)Vl-k1)E#eA=NtW#7s+7Oufz$#@Azg~+?K)x_28%&A;_@hb1kJtrTKKXmrk#GeEeJ@dF* z`Ck$4RRN^`wRcV_`Qpt>SbZG;)?dB`Yp`qRmxfNhv6zlKU#|7HBGDHE;JNl7S{pB} z-HF!v+t=3E+N2vAcR$Oy@nqe(78m37i-arVWm*iB@skp4;?g>7n!iu_Lotm^u_K2EmCHk0Bv2sckj|}F;l$H!g~Xru+T-oYKV{hM zqo|NQe}7$s6fNe-??iBcKh%ypy> z6CE9jeyO5T6lGsTHeykhPZ2^zSVxAqvgivLz&+y*kqWV=k!bp0zkdGpUKRn9+3er% z)fw0gV_)P4SuRXC%SFTxwv+-ngAx-r2gylK7tckmpsy%64Ko0U~UlxPtWBmB=@q@T=>0`Gcm#yXgw0y<$kKps8 zVgP6Bisz?NBwEzDXuMKpxt2E0W(kGh7=M@BD)pX&iY$u+ZOIz zCns?;%gdpz^FD6Pyr0gRD4Vl`r#xSr0wHU;jU0t4Twr0}{Jb{sh~{-iFn)F5kjLzHP;iA+E96#e!`$o0l-qU~@$) zcItq`v~n_BCtbE{8ZZP1l>Je6s=e#JrdL`!zuOb!Zdm~3kFi79<611F(z-OB-?&;h zRf@H1&!urTOEpoUrX1yvJFnOuDR(ug4fJ-izo24`Z|)Y+G+H3Zw8Sg0*;W!gRHzwT z;1(A;$@=m~P}pU4C+#U$Ot-gytm{VmnsuLIUHK3$Yf*S`2SXF~P*t+L*Bdvsw%=Lb z#JiQf|9z|6xQB((#w`n}yQ_v3Dh~E>7vtuT_8OE(A~HSv2X=H5A`b1nqDQ5ej-Dk- zm^r& literal 0 HcmV?d00001 diff --git a/api_json.py b/api_json.py new file mode 100644 index 0000000..6a7b6be --- /dev/null +++ b/api_json.py @@ -0,0 +1,69 @@ +import os +from fastapi import FastAPI, Form, HTTPException + +import openai + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + # Initialize the conversation with system and user messages + conversation_history.append({"role": "user", "content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + {"role": "user", "content": user_input}, + ] + conversation_history + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Get the assistant's response + response = get_response_text(completion) + + conversation_history.append({"role": "assistant", "content": response}) + stored_responses.append(response) # Automatically store the response + return response + +# FastAPI endpoint for chatbot +@app.post("/chat") +def chat_endpoint(user_input: str = Form(...)): + bot_reply = chat_with_bot(user_input) + return {"bot_reply": bot_reply} + +# FastAPI endpoint to get stored responses +@app.get("/get_responses") +def get_responses(): + if not stored_responses: + raise HTTPException(status_code=404, detail="No stored responses") + return {"stored_responses": stored_responses} + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/api_json1.py b/api_json1.py new file mode 100644 index 0000000..8714e4d --- /dev/null +++ b/api_json1.py @@ -0,0 +1,71 @@ +import os +from fastapi import FastAPI, Form, HTTPException +import openai + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(l1, l2, syntax): + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Get the assistant's response + response = get_response_text(completion) + + conversation_history.append({"role": "assistant", "content": response}) + stored_responses.append(response) # Automatically store the response + return response + +# FastAPI endpoint for chatbot +@app.post("/chat") +def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + bot_reply = chat_with_bot(l1, l2, syntax) + return {"bot_reply": bot_reply} + +# FastAPI endpoint to get stored responses +@app.get("/get_responses") +def get_responses(): + if not stored_responses: + raise HTTPException(status_code=404, detail="No stored responses") + return {"stored_responses": stored_responses} + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/chat-simple.py b/chat-simple.py new file mode 100644 index 0000000..41f10be --- /dev/null +++ b/chat-simple.py @@ -0,0 +1,66 @@ +import os +import openai + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +#Maintain conversation history +conversation_history = [] + +continue_conversation = True + + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + # Initialize the conversation with system and user messages + conversation_history.append({"role": "user","content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + {"role": "user", "content": user_input}, + ] + conversation_history + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Print the assistant's response + response = get_response_text(completion) + + conversation_history.append({"role": "assistant", "content": response}) + print("Bot:", response) + return response + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Chat loop +while True: + # Get user input + user_input = input("You: ") + + # Check if the user wants to exit the conversation + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + # Generate response + bot_reply = chat_with_bot(user_input) + # compare this two fields based on fuzzy logic and give me matching fields from this two list \ No newline at end of file diff --git a/chat.py b/chat.py new file mode 100644 index 0000000..69d0126 --- /dev/null +++ b/chat.py @@ -0,0 +1,42 @@ +import os +import openai +import gradio as gr + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +prompt = [{"role": "system", "content": "You are an AI assistant that helps people find information."}] + +def openai_create(prompt): + response = openai.ChatCompletion.create( + engine="tesrt", + messages=prompt, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + return response.choices[0].content + +def conversation_history(input, history): + history = history or [] + history.append({"role": "user", "content": input}) + output = openai_create(history) + history.append({"role": "assistant", "content": output}) + return history, history + +blocks = gr.Blocks() + +with blocks: + chatbot = gr.Chatbot() + message = gr.Textbox(placeholder="Type here") + state = gr.State() + submit = gr.Button("Submit") + input_vals = [message, state] + submit.click(conversation_history, inputs=input_vals, outputs=[chatbot, state]) + +blocks.launch(debug=True) diff --git a/err.py b/err.py new file mode 100644 index 0000000..88bb5d1 --- /dev/null +++ b/err.py @@ -0,0 +1,34 @@ +# # Sample Python code with IndexError +# numbers = [1, 2, 3] +# print(numbers[3]) # This will result in an IndexError + + +my_list = [1, 2, 3] +my_list.append(4) +print(my_list.upper()) + + +#indentetion error +# if True: +# print("Indented block") + +#Key error +# my_dict = {"key": "value"} +# print(my_dict["nonexistent_key"]) + + +#ValueError +# int("abc") + +#Name Error +# x = 10 +# print(y) + + +#ModuleNotFoundError +#import non_existent_module + + + + + diff --git a/fetch_list.py b/fetch_list.py new file mode 100644 index 0000000..3526159 --- /dev/null +++ b/fetch_list.py @@ -0,0 +1,84 @@ +import os +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +continue_conversation = True + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def compare_lists_fuzzy(l1, l2): + matching_elements = [] + for item1 in l1: + for item2 in l2: + # Using fuzzy ratio to compare elements + similarity_ratio = fuzz.ratio(item1, item2) + # You can adjust the threshold based on your requirements + if similarity_ratio > 80: # Adjust the threshold as needed + matching_elements.append((item1, item2, similarity_ratio)) + return matching_elements + +def chat_with_bot(user_input): + # Initialize the conversation with system and user messages + conversation_history.append({"role": "user", "content": user_input}) + + if "l1" in user_input.lower() and "l2" in user_input.lower(): + # Extract lists from user input + l1 = input("Enter elements for list l1 (comma-separated): ").split(",") + l2 = input("Enter elements for list l2 (comma-separated): ").split() + + # Compare lists based on fuzzy logic + matching_elements = compare_lists_fuzzy(l1, l2) + response = f"The matching elements based on fuzzy logic are: {matching_elements}" + print("Bot:", response) + else: + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + {"role": "user", "content": user_input}, + ] + conversation_history + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Print the assistant's response + response = get_response_text(completion) + conversation_history.append({"role": "assistant", "content": response}) + print("Bot:", response) + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Chat loop +while True: + # Get user input + user_input = input("You: ") + + # Check if the user wants to exit the conversation + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + # Generate response + bot_reply = chat_with_bot(user_input) diff --git a/fn_json_api.py b/fn_json_api.py new file mode 100644 index 0000000..0cc6771 --- /dev/null +++ b/fn_json_api.py @@ -0,0 +1,94 @@ +import os +from fastapi import FastAPI, Form, HTTPException +import openai +import json + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + #Extracting the mapped elelments from the string process + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + for key,value in json_data.items(): + mapped_elements.append(f"{{\"l1 element\": \"{key}\", \"l2 element\": \"{value}\"}}") + return mapped_elements + except (KeyError,ValueError): + return None + + +def chat_with_bot(l1, l2, syntax): + while True: + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + # Format the mapped elements from the response + formatted_elements = format_mapped_elements(response) + + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append(formatted_elements) # Automatically store the response + return formatted_elements + + +# FastAPI endpoint for chatbot +@app.post("/chat") +def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + bot_reply = chat_with_bot(l1, l2, syntax) + return {"bot_reply": bot_reply} + +# FastAPI endpoint to get stored responses +@app.get("/get_responses") +def get_responses(): + if not stored_responses: + raise HTTPException(status_code=404, detail="No stored responses") + return {"stored_responses": stored_responses} + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/his1.py b/his1.py new file mode 100644 index 0000000..3cd4bf3 --- /dev/null +++ b/his1.py @@ -0,0 +1,86 @@ +import os +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + # Append user message to conversation history + conversation_history.append({"role": "user", "content": user_input}) + + # Initialize the conversation with system and user messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Print the assistant's response + response = get_response_text(completion) + print("Bot:", response) + + # Append bot message to conversation history + conversation_history.append({"role": "assistant", "content": response}) + + return response + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Chat loop +while True: + # Get user input + print("You: ") + user_input_lines = [] + + # Allow multiline input until two consecutive empty lines or 'exit' is entered + while True: + line = input() + if line.lower() == 'exit': + break + elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): + break + user_input_lines.append(line) + + # Combine multiline input into a single string + user_input = '\n'.join(user_input_lines) + + # Check if the user wants to exit the conversation + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + # Generate response + bot_reply = chat_with_bot(user_input) + + # Use fuzzy logic to compare the current input with the previous responses + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break \ No newline at end of file diff --git a/his2.py b/his2.py new file mode 100644 index 0000000..0d8a27c --- /dev/null +++ b/his2.py @@ -0,0 +1,105 @@ +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +def get_response_text(response): + try: + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + conversation_history.append({"role": "user", "content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + response = get_response_text(completion) + print("Bot:", response) + + conversation_history.append({"role": "assistant", "content": response}) + + return response + +def regenerate_response(previous_question): + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=[ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + {"role": "user", "content": previous_question} + ], + temperature=0.4, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + response = get_response_text(completion) + return response + +print("Chatbot: Hello! How can I assist you today?") + +while True: + print("You :") + user_input_lines = [] + + while True: + line = input() + if line.lower() == 'exit': + break + elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): + break + user_input_lines.append(line) + + user_input = '\n'.join(user_input_lines) + + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + bot_reply = chat_with_bot(user_input) + + if user_input.lower() == 'regenerate': + previous_question = None + for message in reversed(conversation_history): + if message["role"] == "assistant": + previous_question = message['content'] + break + + if previous_question: + regenerated_response = regenerate_response(previous_question) + print("Chatbot: Here's a different response linked to your previous question:") + print("Bot:", regenerated_response) + conversation_history.append({"role": "assistant", "content": regenerated_response}) + else: + print("Chatbot: Sorry, there's no previous question to generate a response.") + continue + + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break \ No newline at end of file diff --git a/his3.py b/his3.py new file mode 100644 index 0000000..ab1de46 --- /dev/null +++ b/his3.py @@ -0,0 +1,80 @@ +# his2.py - Chatbot code + +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +def get_response_text(response): + try: + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + conversation_history.append({"role": "user", "content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + response = get_response_text(completion) + print("Bot:", response) + + conversation_history.append({"role": "assistant", "content": response}) + + return response + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Simulate an error in err.py +error_found = False +try: + exec(open("err.py").read()) +except Exception as e: + error_message = str(e) + print("errorrrrr: ", error_message) + error_found = True + # Use the error message as input for the chatbot + print("Chatbot: It seems an error occurred. Let me assist you with that.") + bot_reply = chat_with_bot(error_message) + +if not error_found: + print("No error found") + +# Continue the conversation loop (optional) +while True: + print("You: ") + user_input = input() + + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + bot_reply = chat_with_bot(user_input) + + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break diff --git a/his4.py b/his4.py new file mode 100644 index 0000000..2354a9e --- /dev/null +++ b/his4.py @@ -0,0 +1,83 @@ +# his2.py - Chatbot code + +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +def get_response_text(response): + try: + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + conversation_history.append({"role": "user", "content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + response = get_response_text(completion) + + conversation_history.append({"role": "assistant", "content": response}) + + return response + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Simulate an error in err.py +error_found = False +try: + exec(open("java_err.java").read()) +except Exception as e: + error_message = str(e) + print("error_msg: ", error_message) + error_found = True + # Use the error message as input for the chatbot + #bot_reply = chat_with_bot(error_message) + + # If there is an error, fetch a solution from OpenAI + openai_solution_prompt = f"Solve the error: {error_message}" + openai_solution_response = chat_with_bot(openai_solution_prompt) + print("bot: ",openai_solution_response) + +if not error_found: + print("No error found") + +# Continue the conversation loop (optional) +while True: + print("You: ") + user_input = input() + + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + bot_reply = chat_with_bot(user_input) + + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break diff --git a/his5.py b/his5.py index b81d511..4bed3f1 100644 --- a/his5.py +++ b/his5.py @@ -1,5 +1,6 @@ import openai from fuzzywuzzy import fuzz +import traceback openai.api_type = "azure" openai.api_base = "https://cymetriopen.openai.azure.com/" @@ -19,13 +20,16 @@ def get_response_text(response): except (KeyError, IndexError): return None -def chat_with_bot(user_input): +def chat_with_bot(user_input, code=None): conversation_history.append({"role": "user", "content": user_input}) message_text = [ {"role": "system", "content": "You are an AI assistant that helps people find information."}, ] + conversation_history + if code: + message_text.append({"role": "user", "content": f"Here is the code:\n{code}"}) + completion = openai.ChatCompletion.create( engine="tesrt", messages=message_text, @@ -43,22 +47,22 @@ def chat_with_bot(user_input): return response - # Initial message print("Chatbot: Hello! How can I assist you today?") # Simulate an error in err.py error_found = False try: - exec(open("java_err.java").read()) + code = open("err.py").read() + exec(code) except Exception as e: error_message = str(e) print("error_msg: ", error_message) error_found = True # Use the error message as input for the chatbot openai_solution_prompt = f"Solve the error: {error_message}" - openai_solution_response = chat_with_bot(openai_solution_prompt) - print("first_res_bot: ", openai_solution_response) + openai_solution_response = chat_with_bot(openai_solution_prompt, code) + print("Bot: ", openai_solution_response) user_input = input("Do you want to continue the conversation? (yes/no): ") @@ -83,7 +87,7 @@ def chat_with_bot(user_input): exit() # Terminate the program response = chat_with_bot(user_input) - print("continue_resp: ",response) + print("Bot: ",response) # Use fuzzy logic to compare the current input with the previous responses for message in reversed(conversation_history): @@ -100,4 +104,3 @@ def chat_with_bot(user_input): if not error_found: print("No error found") - diff --git a/his6.py b/his6.py new file mode 100644 index 0000000..2d3eacb --- /dev/null +++ b/his6.py @@ -0,0 +1,103 @@ +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +# Flag to control bot response +continue_conversation = True + +def get_response_text(response): + try: + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + conversation_history.append({"role": "user", "content": user_input}) + + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + response = get_response_text(completion) + + conversation_history.append({"role": "assistant", "content": response}) + + return response + + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Simulate an error in err.py +error_found = False +try: + exec(open("err.py").read()) +except Exception as e: + error_message = str(e) + print("error_msg: ", error_message) + error_found = True + # Use the error message as input for the chatbot + openai_solution_prompt = f"Solve the error: {error_message}" + openai_solution_response = chat_with_bot(openai_solution_prompt) + print("Bot: ", openai_solution_response) + + user_input = input("Do you want to continue the conversation? (yes/no): ") + + if user_input.lower() == "yes": + while continue_conversation: + print("You: ") + user_input_lines = [] + + while True: + line = input() + if line.lower() == 'exit': + print("Chatbot: Goodbye!") + exit() # Terminate the program + elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): + break + user_input_lines.append(line) + + # Combine multiline input into a single string + user_input = '\n'.join(user_input_lines) + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + exit() # Terminate the program + + response = chat_with_bot(user_input) + print("Bot: ",response) + + # Use fuzzy logic to compare the current input with the previous responses + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break + + elif user_input.lower() == "no": + print("Chatbot: Goodbye!") + continue_conversation == False + + +if not error_found: + print("No error found") + diff --git a/history.py b/history.py new file mode 100644 index 0000000..fb0e2ad --- /dev/null +++ b/history.py @@ -0,0 +1,73 @@ +import os +import openai +from fuzzywuzzy import fuzz + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +# Maintain conversation history +conversation_history = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def chat_with_bot(user_input): + # Append user message to conversation history + conversation_history.append({"role": "user", "content": user_input}) + + # Initialize the conversation with system and user messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + conversation_history + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Print the assistant's response + response = get_response_text(completion) + print("Bot:", response) + + # Append bot message to conversation history + conversation_history.append({"role": "assistant", "content": response}) + + return response + +# Initial message +print("Chatbot: Hello! How can I assist you today?") + +# Chat loop +while True: + # Get user input + user_input = input("You: ") + + # Check if the user wants to exit the conversation + if user_input.lower() == 'exit': + print("Chatbot: Goodbye!") + break + + # Generate response + bot_reply = chat_with_bot(user_input) + + #use fuzzy logic to compare the current input with the previous responses + for message in reversed(conversation_history): + if message["role"] == "assistant": + similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) + if similarity > 70: + print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") + break diff --git a/input.txt b/input.txt new file mode 100644 index 0000000..255244b --- /dev/null +++ b/input.txt @@ -0,0 +1 @@ +l1 = [' Id', 'First Name', 'Last Name', 'Band', 'Employee Id', 'Business Unit', 'Ctc', 'Cost Center Id', 'Date Of Exit', 'Date Of Joining', 'Date Of Activation', 'Departments Hierarchy', 'Direct Manager Employee Id', 'Employee Type', 'Grade', 'Group Company', 'Hod Email Id', 'Hod', 'Hod Employee Id', 'Hrbp Employee Id', 'Job Role', 'Job Role Code', 'Job Role Name', 'Job Level Code', 'L2 Manager Employee Id', 'Separation Agreed Last Date', 'Separation Status', 'Total Ctc', 'Candidate Id', 'Center Type', 'Company Email Id', 'Date Of Birth', 'Date Of Resignation', 'City Type', 'Dependents', 'Education Details', 'Emergency Contact Number', 'Emergency Contact Person', 'Emergency Contact Relation', 'Full Name', 'Gender', 'Location Type', 'Marital Status', 'Middle Name', 'Office Address', 'Office Area', 'Office City', 'Office Country', 'Office Location', 'Office State', 'Primary Mobile Number', 'User Unique Id', 'Work Area Code', 'Current Address', 'Current Country', 'Current State', 'Personal Email Id', 'Personal Mobile No', 'Office Mobile No', 'Bank Name', 'Bank Branch', 'Bank Account', 'Bank Ifsc', 'Bank Pan', 'Aadhaar Number', 'Job Level', 'Function', 'Leadership Group', 'Nationality', 'Function Code', 'Direct Manager Email', 'Direct Manager Name', 'Permanent Pin Code', 'Latest Modified Any Attribute', 'Regular/ Temporary', 'Blood Group', 'Departments Hierarchy With Codes', 'Notice Period Assigned', 'Salutation', 'Permanent Address', 'Permanent City', 'Permanent Country', 'Permanent State', 'Date Of Death', 'Anniversary Date', 'Group Company Code', 'Functional Head', 'Employment Type', 'Contract End Date', 'Contract Start Date', 'Attendance Shift', 'Role', 'Dependent Role', 'Cost Center', 'Office Location Cost Center', 'Is Rehire', 'Rehire By', 'Rehire On', 'Rehire Reason', 'Employee Separation Reason', 'Admin Deactivation Type', 'Dm Pool', 'Home', 'Sub Employee Type', 'Hrbp Email Id', 'Source', 'Source Type', 'Fnf Status', 'Past Work', 'Passport Address', 'Passport Country', 'Passport Number', 'Passport Type', 'Federation Id', "Father'S Name", 'Job Role Alias', 'Channel', 'Fls/Nfls', 'Short Desg.', 'Zone', 'Region', 'Recruiter', 'Zrm', 'Hrbp', 'Zhr', 'Uan Number', 'Esic Number', 'Pf Number', 'Activation Timestamp', 'Designation Title', 'Latest Modified Timestamp'] l2 = [' Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] find the similar elements using fuzzy wuzzy logic and specify which element from l1 is matching with l2 and i dont want code just give me result. \ No newline at end of file diff --git a/java_err.java b/java_err.java new file mode 100644 index 0000000..08d43bc --- /dev/null +++ b/java_err.java @@ -0,0 +1,37 @@ +// import java.util.Stack; + +// public class StackExample { +// public static void main(String[] args) { +// Stack stack = new Stack<>(); +// // Attempting to pop an element from an empty stack +// String element = stack.pop(); // This will throw EmptyStackException +// } +// } + +// UnsupportedOperationException: +// import java.util.Stack; + +// public class StackExample { +// public static void main(String[] args) { +// Stack stack = new Stack<>(); +// // Attempting to use an unsupported method +// stack.add("element"); // This will throw UnsupportedOperationException +// } +// } + + +//NullPointerException +// Exception in thread "main" java.lang.NullPointerException +// at com.example.MyClass.myMethod(MyClass.java:10) +// at com.example.MyClass.main(MyClass.java:5) + + + +Exception in thread "main" java.io.FileNotFoundException: file.txt (No such file or directory) + at java.base/java.io.FileInputStream.open0(Native Method) + at java.base/java.io.FileInputStream.open(FileInputStream.java:219) + at java.base/java.io.FileInputStream.(FileInputStream.java:157) + at com.example.FileExample.main(FileExample.java:8) + + + diff --git a/json_schema.png b/json_schema.png new file mode 100644 index 0000000000000000000000000000000000000000..967a25c87065fb89a77bf718c85b0bad2ac44860 GIT binary patch literal 50232 zcmeFZWl&t*y8Ro1CU_taoZt{7xVuBp;2H?--Z&u;+({s~yF0-N!GpUt?$CHcBYhk4 zzI*R;{%6s`zKG} zF;I|TTcE}!FJb?|Ijc&GJt-e1-i3WYFc(!2ee$F#8uiu)5%&3+gN(NGlP7r84?l3S zs#M3YjVYFDS}yPHEFBa~U5reOT#R7bo;>lQ_-JG4plokx=i*Gkr|U)GPQk~-!Sj}x zhli7ui;=ekjIoNrbSy?%F|C6tIIM|puIe2;h+V{VW{f}?; zeSTbI2tRq!MI$RAs^)HRaEJa*P4gc4zTFQhlfbIZSpQiCSvC-zN?f)5i_{nWnrW_Z zOohr$1g<6ZdAiF2TTX4xm?zI9EY`$b-`JM=$h{uJjjOYoFy^*RvI^AaHhK~9;b#CL zNf{dl17VU6a*)=eJ5A$y#2fNcbaoF*mLV`8WKO-cBF6iws4s16p zB;SXfe*S-Srx++LToE)Q-LG`wq|mfZX7*JoNt zvb`|t2?flIHm-lPVHbi8ll@l0HWIpX`TT3S0o7-rmAykz&20sj-4WRO&C`Ri5LX$jz^=O$O;dzj88|P#ao&$dY)_W#kGH=%hb!n+X=ELEk z)$jegODSdt^|t0TrdexvF|aF3_2D;(WpDA`2bEFFr0^n7caKnzitpG%o@xv0&tyET z0F{Jnq%fvsQjv_ND4ByHNc3h?w?`UwB)F_9GtP$HO${ndoM^YYci&)%m{?K1Jqb0r=tv?17qBY}uM04GHAYXY`Zxie>`PMUe6bV9-F6hp6&_Vy9)+N@(D;Smfw-!q4rey{U7rY)7~`zv=HdeTPeAeCjBqv@HVQuWZ<(+GD!j zDb&R%k#U-97a8cct6N+z9hmzZ)Xpd-5Hxmi?Yp4|$+!>vnHXH(RY5`W=aQHxTE`Cc z;nSvRv*j~PwVl;-em=y<87GJ8_7NyIr?7=Z+`AXE?QY_Y_58$d&HqZ+{|NMUTc$*j zL*nfIbr>hB_BtS?w^rJYXLEm)e33zv5C~`ACC|1a?D$%|EZQ-{_Bt`5K@!)ixhF zfjVi7UDIQB8ENa!DUJX_3E#RC$F(+QI|h=DvcrSB$_1y=n@os1ffkUeAf zIkk1y0-6Q#NWY5BkP+D&riy$R&W^3efV5SYiw&y9gppc zWUy+=htJliO7V!2j|!uk6xDX`F5nE>%i<`ug6r315;nDO57&tJ991P@3+&m{ZL$JD z-`yLJ0UJ|N=mmJb;I;FPPVIuo^&D{6xPLzSAaN#yZo2SX!>sUT`PGidp|rV*>y4j` zQXKcUpp_!-^{z8RZfP|vsawig@A&25H;Dj}-BMf}UKNT!xSP3rZC=<*Q>DC=&` z1WUytzwJ_--TR|PSqn~Bi3klHxpI{{eUIULbU=ZKhRE{nkwIh_@1a_x|c#D{I3c&4F+aC1A6Lg6SB%*!yGe4!dPCEpqy< zqBEZxJI(yUynybKab=lS+n=@Y?h00~|A^x@C3DGGZzUkGpns#W z-^h|K;uO%OWf9R~udfo@YNPEC{2tKN6wMCtdni6@aY>5_qe zHir+56Ie*ECPVUPM;}hN4AF?M)vR_TXL$GTCB;cH-q2QXi|K1{wEg;d&^1h1}mKbi;u-QkyDrb@Savvb2siXOE#cX!s$ z0$QaLRW5SI=ooz+_X-Z_?S!|_oYZjd$S0=60)|)dMO-xs#im0YiR zy~uc95likjQV(twH7rIy0Y_-|SDo!RVRTL{_(O0epn*O0l1s#F?(gN@e>0l{)b~m;#MabjA=9v`q>$URBX;tvUC3UB^wz|N>bUL)M)L#C4(T{ zI!`{m$cg5JE`|yomyF(JEUVUwR?Ks~+*j?i$5j=*(DdUAaGIo|0=g;w_r72}pK%Ew38u!CmKDNQ*?DzE8+uoODXDPP zmaNqG75Ttv3Cm6p1Hm#gA(NJ;4J7?WMv`D~RQU>NzCOXd;9G3TF450hL8YwX{U#MXX=Gx>oHUtnd7{x{v-;!= zxjEDbu{|dP%?gQg6XTT(2`{2c%93h0g-cQ%QkX#lcY(} zgJsW3MiS0E{nfkn+qW!x^VgTxm(o~>rNt%d&`wJspQ;aSds9jTdGM(|-OUl^`z63A zi*|%V#U*lA(Y+8hD+T&r1^XE@APVb=uove=Z7io^ZE{i)l&r@}apTkkoxwb5vHpwH zOxKvRL9VEI`<9G?Jj(&j@$)=F`oKUc1sAoSOO?*sV>s#B%{2_BqrABN4LUQE0Yn z+6o6P#Vp~?zNi^HHA}$0N=qDjRmu|Jfk@{mE3CriC1kLCOgjQhF5EI?-e*hpA5UyO zeD~Z&(qf6aet(tDoC2r%-9fw>)o1_)GscGN#T&Y`=I?1o2tJp7wHV%?PcB)>3DU4V z8U}p;C@0R{{l6_n>_>E?-)#1Th8w#{G`S8m>r+VsVc&!Rj_g_BK@>*d$fRwR7ql&x-k_@Hg*P1WAL(?E>2Z#2CcS%s&(i7kiUc9iL*YZeQI&w7>t#3Jo#A^vP=rhm*i(qa+l8M=N_VDn&uAohD0;+n~f%# zjN*3!J#Y@I4-N>ZcjOyV#tJiNIy)3ntsz1dqls%wLN}df`eLc}P2XLr)%>b4z7>>r zQiF5!FvK11x6NBLnd@aSn{X_td-+;6t$WZwvgp`fzHr`GCUJHvTMbLRu6(RI9Oj}a zH%)A!KLB(lAb(d)ub-`B6Q_pKxt*v&wY;VBP;DkJXQebGaD{kl&yEAH7Dl8w&CU&;QpJ4|JB z1!Iyl3YR;!rYsP~o{c#Q`k?eup$qe=|GX2-+FDRc?yD!co#tn(xJ&Gd{)|6#EhSME z48HoNG156VN3_tD5k!-GW_!DEDZb!H1)}Rs6(q}amkZJ~^PtS$#fkR^a^_apVtTMN zu03n_7Jiw{3v9HNkXVO()9%;7^Q0NcSG^)WZ?c}0i65{!eQzbeQ(AD>?6_HR?G7+m z;3H#K{_sXbSNjW~b+5sZ?_<=xcWziBP+9!DXF|o&mwH#0Oc6iQXE^;0>b%sAdG-4w z-f1tcr_ZwNQfyXs`rl*}F-fx=zoI;aI3U{D61UoRsD0z6$Ua#i7c_dSYbjj+vSq}f z?G7$CG=JF;4%5sT+wqcFnZn)Wy zGzGcF0E^a%D27RQN8T7%7$fxlfb1JNg5{Kw1g?U6v|lyavj6(zGn)ndP~PU^n3e*m zcj|p5*5d6gzxkc9pHG{=8iXWOd~Un(MBaPzCi24VTTYQ~Id1Uz=L`24EoI)Ox}PWz zwhmCSc!EtOU;~~CWb^5+TqYysOBi>}RP{swiJjy9+KB~3x7zPAvX8P(DdH~s)CvvV zemB#v;r(F>?-qoApsId1zX}UmV`*?Ou59*g}|DKPd$=D$eKQiuxKKG zW7flkH+W_5B`pW-yz3s0i7d7TQvy4{VM`>GXaMl&GWnLht22MmVwAQf?3$lCFk!8*id6g z@{AC=V%WMM=@t`yLd4`h*`>vIuo!iZS}+Mmz2kM;e?er!-u?U7wU$4kc7(5-AUd9D z_}5sdzQlKzf6)#S_p%bWgl;SB7>3YRv>)j`jzVe2XqcUcyvv5->**}6%EMCDht^W) z`pRj?SEbu+{DD_LvcryM=ECRvqdCxKE(}+{3bqX?Ue-Qy0Qd&4MeCddc$!xZ5BBb+ z4H``WU)I)Se!_4Tg9_rj^dY$lj7RqEA{gAv-jiLHQQeY=ijkB#g;$Dfk?dxyG@^g>|Qc)Y*5D|ro^;!u9mc0$|=(L=D? z=1k?k#4NqfC1U<4N>)}VJRNjtP&F)0ZnWy1e|!o(WX>x4&6g7MMtU5f~JRBNpC&_p}23irQL5|b@c~8;#~^`s86-iU-fOi zMcDT9?eW?q`)^iT^F~s# z(2$;BPTy>ckJ`C>hH}-7tG6Q_#HFPlM`^P|j3OUR=E|P`*ngGeT_`mvtqK(S%8sq_ zmi3w#`skg$WdB#=tGs6v@VD{RxM9;7(9Zb$m1T6kjs(~;@bOLgU}^DEto*UG-u8Ou z%Y<%Ki*peCc}u3j-jc8OVQC7 z^5KiM)%`fp^`z^mbr4TyDnh*Gayr_WuK;;i?RT`>hN0JL$Vbwfvs-iW?_J3bW4C(j zF@skwQv?D=kaNQ=JG`#K-Nss!Xsd0J+b=N8Z5~~w`HgQ4wBog7m?0FAkFQTZUvtm~ zv0jci|LNr<`5b`HNjnMMxT0)8!A@s^Q@%oWR4) zwg=$7IA7>xBO>2z7;kK+eKi7r-E~pXRSL9Tra7)q>&npf1)y4ZX~uoQ#2v|!L1Z=u z>v})(T>6T7Zpoo-(0iNp^b4+B+Ff+w#%EsQ@Y4nuLVJrLExzt#`t7%ox^-AX#WkGr zxBa}S@Z1%ug=5&^zb5<|P^CnGj=Rw;c1_9-{OvtoyM}&nUGwAUt=`fhj~2_lFv7ud z=vR_#T5RcUqG)ZFOw(;7lfi`cxE3grp}7uL#f!j>4*UAc#GuV*p9SWBi`>3DdqVTn zHWHy@X{_aoa-q48ljYa#(HnmNO zrBse0xU%JQRFiNA14en&h;j`8wlW zDl-o95)~_@q5-7Y?+gDCf!0Ji_o?Ss5nLfqXymvDlCw}#Ch%Rf!v1n++slz1z_Jgo z*^1vR{LPffhFPc&wUQiJ*xd?!PC(_%lu=oW{@d77n!qN%QhG#-xjI~@muQQkGp<$b z=CqVPS&NFDxBLNVg|>|=&vvqWqq#qEuf8)D3S%4c{siXcTaX|S;;cr)fpv-Q9WDO5 zX`>#rDBKn~L9`W(c1wV0p+6c4F=Wnh>}zg;`9&vSrw=cCU&H(&DsH{$4EHZ;3*S4y zbn7U$TB?D%{A}HD0F=XE>qHABC{3l#tOhnjZ(6p2f*74kt;Ve z@=O<*r>RbixKh${#otFN2nC^=!EliTDsf61Ywb6xsvmhU2Z_8|aHhCwpfox-#fq6} z-L3*3YkSkw=xpeQ_i=lS@3py$eT5IMiiuVp(B-+=w#M!-#Dmkz#}7Xzb7``beXR(J zF;z-^OIjk9K9c|COweCbPLIU=c$u~*G>|h7tE_ngfXLD%Q56_@(?jujnStQk5!oDr z1A^$&e}rRmr)MX%%o}X@Hi4L1G3!2CSrN~q;GbCl-Yl)rbCgtP}(1&Ik7X{$S$La5^d^t?$Sd&EQ*r;$*AwOZfG%gi;H2; zaWj}-My#pybNS8o+J_+-bxS#f0snnt9IpLLmJ4ta&{WhjVFZBPO^$TWYBQi?qNj7v z7P!{C^afMZ$NuV&aRE-?$9Jb?F){wm~zsxio5(uuPwxXec+ze9`DPrjeP0z}Q-bw%#A82ju=C<kNxCI{+$0|!6;&~H|H#KvzNbIug~iRs@|X_=!ViSaJMv$kpQ`$m zHzjigmSccq*I~ysx)PGfV#}`@#mhe+h<+7f{1P5YrTtAM@J(=uW33DN^hQ3VAo~=5 zQe-m4hvGKrOSn-NOPL{UXG780WE_9L-I_I^5s&XJ{(+QgpKmj`Bz?k67zE`PnDD0R zd(*+uDz+%B&NMS={Zqc%qRh7!W!aBKyrij}MX~shHUQ{s&P$$*3tNk~87B(gNm?#j ztm?t5@UfKqhrtn^0n2 z8O?0#EuLHJ0mQQ|ueIiqss*;0FWPbMBdLy109=t64CK@g$rclbKNh^h@OKSGuTL_T zO6S;_Ud(XSV<(N*OAXD5ZP8*yvX&E?9~c;IsKL7F@sNjDa%gy{$)KI-$GNh-a z$;eF;32IAX^v+lf00?@A*!d;Ifgc~-uBou<8cxg<7ebZVkFWziV`)m@=Z4Q?VNtVG z^)Ksl7w9Xf@WQO=VSBW+I%lk8yH3iQ$C}105QPbQ(bw&Dpn4JOq0!Hlacn&EnjNYs9}gTy!)E? z;BT42{H=pG!=t+rW04~#(E-4-&gyua9SjSTZJxipBXSmb&-C??N8KB#p`12-?HkFXLB$(jX=#u)}P;M|g03XGf}AA247)GGSirZbTR?L#UY21Nb5 z`Rp7YRq?HEmhSfE`uXr?PjVp?og4HXLjbBornyY{8wwR8 z;+jb%yhcD(`DzsDN4%#ropn0Nx$a0lw@)0jayv5OUZF>9GDT_WKd*A@i!8lx4n)Io65n3Ud{hSSmR$aRpkHb_V7*A8@Zpt*CHD0Sd1FLS;jsd_Tj%9IqB8) zzHA`apwhm4RP#7oN2ho__XJI!$cVugM04TkXb?8{{q_tFvie2NSxqoT8`q69R(>0C zV8Mn~Fc>-;K~P*p0lV}fGsL!be*!XbkOzrW@+R&gRZ#Q3E7#iLN0s|dH}c`-?(Zyq z^(JPT;=G|d;B*vF4A=21FT9AJW{2$!)6rrznNL9dnJniGsU*}pI0O;Bf^gd`P&7V+ zz*Z3BuwFtU0fPLma-(-if5~^2g{HSV6Y>8389hVw7c!4LhO<>2Z zB|cJU$`vvtZpiI>CRqHm=a@Xq4DI&ZU8w863Hd1VrWfRRytwIDt7o=C8(`bXcA%RG zzp%zDb8K76BS58y;Q2xLLNKf%$~!$Hga-K!%d#r?pAjomG-TWS`YN^&*mhAGX8qFA zOHQ2it%LW7s)}wUoB7E=x=%t)Z4AUSgQFH)ut9CY08NjU6kzysoJuC^|H^@bUL3kA!Gj z9`2gUrc`NvDxyFOW^X9FMSF=bB}F#&qs~9)Jbw-PVkI$WdSQY4?RpyU;&29e%(WOU z1$-dA%z}TM@&La`g*|qF5twH>wBQ3wa@2t=M_23Ye73nc3m1Z$+%>LYS%2AV2_7Tb ziZ_7u0mHMCqdqQgqZs(rzV>EBY1c3^-j1l-mTas%zJASzmTZ=!;DSaos_njki2uF7 z=+&8R?xWGxwhlAe*u)E@diFsxug&$~kzu&QJEVJ9G5WnGhnSwWHL+ku<68*_bo%VD z(vYjTc6w1$Id|)+mVu~Uaw0MYf|kt?`D0FkCH|xd@hi@`36FNUY#*1b1`t|6N=Xidy}$2 z=b#8MawQ)FbP{a@ie8c$yvXIiWm1;)eQr=+)I0ZfG%{aah-m>C)Zw^?P=8qdD^*`z zX8Ap3KEuXTU6x_G{BOO$&ldze)$b2>rk`)ngc4k9Etm6|^!Efx3yGW#Bh7B!o&3-) zL}~P7Wj%Dj+|}MyaS?zMk-!aqb=B)shwXN8FwfZ1=wy<~Jm9I(%Exk#8a$cm_JhD< zo7jT*6w@h+s6g4zQF*ABR;a+BVDix>8(Idf>)pvt5)}nWCsx&Gy-(ZuWzK}|dqe(? zx_3BdkQYJI9(hlxc3+KxBH0Sz4KKtfj5F?y{n(~07nf~@=sYVGq&UDc{yK#IVG zDpipg`lE^pg}K{5JA|3dDBF2vHSVANU|=F5O3?YW|CShE+Mvnc|I|rLU-uPi=0?Kf3P%lXU zrSFzl_c&p*!?#a3e>`z%Ubstxav>&V8+kv0fl7=>_a~9Lw8_~*|4J9G((pKbCJ8pH$aT-u0`NJK^Rq9la-+zsbo_1x~0mM4q zmTk%?Rg)Vz;>mHJ^7>L_@4krdIt@~~+yEy#h8pf&(QbLaI2hxo(ks(aL^hc7h;AV! z6Q0l-d+c6Gy6PI(36B?#=T<-sg7lOE_&}$xNI&axHpZ7{IgTXG76Hp3>--k;$1Nvj z?pjRdrQCt{;SDjzsthlF8@WX0eMn1;p5RaHvPyqdc(E}THUe-@(F9r5=ZDME$_w9~!!7fVeMDO&!lxBN)|+V_Rcj#A z7Dn>3D~Dccn;gG2va_0~GYhq#S zF_-`QrqjP6AZ5;7=9)9He41-w$jz4E0i$=eIH$fp?7XEOpfGGf&igFuoxKCmnTyxI zMtn)@BX!3#KSX#uAY-xE-vZ8O*_K7*E&0|qv1<8+dbs}3bvWO^S0st;flKt;aYi*o zR1Y}ar$XCBfS1TU5e%-Gd_)?b{7-?Jru;J`#A)3|&=7EwUG}1;X>3KVo_k@dA4)gp)Q)Vhvm7?hLA$9n8bH`U^2Fr{Po{5RzhN9pZ?h` z{)(#44bd8YL6*n}jE;+)dH>?dKkg%m!>-5g?^*N6Sg>vF1vJNCa9W`F3}i{O`6P)| z9yYE}HrFxjuiYeRh$|Z6oBEaRIj19`114`I7!^qOy%6no`m5LmbAmHq)JK~d@lWrs z=iA zuO2vLo}_?OT*w`+!FSiFSMcFNz?}*>haInqi6AEo z(18gEn}1K(zL+Y5fXfy>XTR*1`A*+_(j3%70yAh&V8?^~<+YXdt#pHCrppBd_BouU zJ!TW1eYU*c!*Z^M4imPQAz&`cA(Dpa_(m3PV(z)XE}i6rcq^3c)^?`ddsr>ikp@iW|#<$?bZl^68V&aGgvo`zxhc*8nz?xnZ*$yy_ zPZN2p%ea_MqUc@aMvizJ{(D2CWF0fc5~upiye+7txIV#rf>6$TOXJhg{L3yp;wk#| zmw%y|9^YyCYnlMahf+@?DUt`_RBB;3AILY?ob<=)*9!3fhZ2PT+v5|imRgh(D`t#c zUM^0s$=d+y6*Yg|DNtsa0|A%h6@^rH(+il07txJU{EiSiTM$p3YFk5NmYH>Bdp@NR z;&WC!=Ao{NA*6m|#jVJq9Io=of*Dz2jDiLpdI%@T?m#x=$)#3Tz)XOguzFMJcM~|> zMT*gBtJvijl5r<6?mV-S$pd%AN9{Bw>}AOiXV$w5h^KtSJDY4v5)v}&s$W?fBu57( z30-pul!6m=4_MNqe6xQLV@aYu3SMM1bNgsk_MrPVvs+(j-mQcA=sF)IV+CWZOG-|e z<(vZj&AyTw0(~|zGok22Aj7Zbp3h2!qu||1kKgq}1HGN?c@(0*oA^e;<12M+EShqJ z_V`rtz%$71clC1Ftdh6JZ1v{M}vbPw|%l!?f_O%4igQSv30U%Fc6B9(JY=p1Q zbY}tb{P{VXf7GB?^Nmjeu$@Tr9wHax1%mjtTJ>?jBi4(#sCl*TYs}$sFtX}UL@Ydb zWrV84zUC_QqDP>g_bW!ggiuFLw=8Rn?zKjRt((6|XNZq#rjv_$IE`baG5w@;sCV$rbzuoq|$RD=c zMo?DLA8#^VTKczd3k|D-qF3_luwyRUky4f=TpmTs4)^O@`$h2Igo`M_6=Ke!2Nh~T-_-$pFFxhFr>&+QK7 zsIu{U&s6OT$*9V9XnI$g@>l1-MC*?`U(eJBC8HoDL%M(V7T&9}!|<`T$Mcsb&SbDw zT#FG9g7l?bpdAD(2c$k1siSxj`f2}2FuFB(WR1Nwit_}K7}y3kcS`jW^vN7aE&%As z&Niy5Q9AAT*q#fn8hzytY|2Hkg}LUP>Ik!P*dK;;%%tFS@Lh8569-`>#r>;=&leW6 zN$@2+-8SO^)jz{P9lah;UU zX>)pi>IhLFINub#Fozl0%-ocq)Gp8m7GAqlu{+5ePyCFlRL@9+Yp?oN5IRKFHl*FW zMJ_5$I?)-Wd>LahuCdp7z4ji0zA08 zMv79)nNwvD1g#g~T0kiox`^|ki`lDv+VLm*A{aQ^z&>|2X8`geL z?iu8cOe&ojZBQqzyb!g;B^r;wLDcL;R4e-NB*1)i=Nr3YrCFbRXPe#H?^uo7Okk1# z@+84YjOM$$A+e)Nw#DM9iB+rAj-UY5sfC{4y7 zr0uzZ9B11EjFI@z;8K41inh}o!%}_{4g1R{7^phy2T`8_yPMkjC-`lt8)m)cQI?AIkcv7YKV{;R^GFOLeIE5WoH(f0@JJ1v7esy!4 zqvUW-9CJ9-<{C$XfZ%uXvAOeMS}||457DtBzxeyTIJBLOE*(JPnDbg8#*!{>)Xttx z2pDP}bhP#o;@aKpu{KB*A}>Z#|BDbk2p9QRLNx0?B}AP(9|%zngjRYcN_3gpLu0Ym`ElO~0lJL;({uUt5FmEm|xVn}OG zHTlT5DiRF7;n7MiVs3BZ8_ujQo%i}lUQkg^FsSlN+kCJ@JfN|RPxHy1(q27jHT9iZ zhP{Fc@A03~6^O|u*j7b{jo8&8`~?!+R8_VLKWR9bXoK&Cj9vxatIhL6x!1dCxo~%^ zN+k$fU&959n8|bUeS@87CE*{R!~-Dr6=v)e5)rSLU!(;PXum%l)6fWNOZzg5e@qN< zsVOdxf+81OpLai1S(vJMmG?3E5gA>EA)_|!2-pnBAFoT-lgBC@9ciKzSNn(ZUsCV9 z_p#ad;>{VgBMYnSW#tl05XhUbhZ&$i5rMp6cqH)@JG9SL** z8->XU&1dG5QSO(%aY>w&NoPhW+e>dtsL^{Hj->W`uKB2WdeJGxv>)vC_)!L*?5OrS zJ$(h)M{7K`2|$|ev%f4N-XpfL2QEJBt<+t@Y@jyF*&Je|JmzOXw!h-4cp}NNODktK z(mP0Wzj3XL!`12Z)=rb?1k}97`4@Z=6I3NhaBZOv{_KnYZ17i`lH|{KDi~~rFKLws z_|kTEVe2eAjf6lI{Kx@-rMS^LCI)AQ%ZsMwKaMa>D!qS z5$R3B*$wBdVfjMH&eH>{FZ3?EFNiTWf7iJGXc=$TclGNI`mOB%M7Q~|X%8xb64G+A z;~R=2|Q|4Jz@;2Q+S~o=DresX#I$6cd zgqAm0DNCh{H38*5{aHeQvQ3F_wC}aPY|?BgVUH(NxUYmjT0XIhOO7RK^~aOMqY1+> z!&JO6X{pfcXYhVh2$t3}eit-)PE?$~JamT#r8kj|Ivim>;b8l(w-aoc$=Ycz4RDJC^P`TY!nWWzuHGRP}hA=00 zyi%t2h&9Af4k~<^n1Uq(nH0?^7Wt`EIE*5N_0I5&(u-pxJ&$oI#O<=jUX8m3C@H-A z@@S9mHC@B1Ml{IzIjo5#4`Y*@#&3ya`tN_rCla=XyBdo5Y}nbp@R&V!Xi+qe@sh^xqv~@0Hl9ZkkPa{=Pt0` zQ!vah{?HX>LgoJS-p56sFB-Ci>Km{Ns~siz{|aDw{b$)snRi0NfTryER~H9%$d?*v z60Ou^B(8%~TIT8{*SiXtWp-$Ms;_>Z)br;7S|4i^kVJLmGE~%T)CuE4mdP)U_$AzgUkuB_A_BeVTe&+23fG+3h!) zjqBO4W6K6&oCBY+b{o5SGwMmBAUj?@?Fdm* z5Tb)lloC4m9a8!HZJXb)nrps{15bt&J@;_6-APO|G-np;grI%Y7CuuS1sH-vd`$DI zmdl+uTnE3c#TZnQf(TI=q#HC|!HFRS=)3mCp^f zX&!UerM_;S5AfwYBOXsEOtPH#Tj!B0S_m|7=cmNZ2=4%Bl-{Oe+i!q)BHENdE5T;L z=`*Bm+5OwqZ56bZ8861-AF@U-O&P_-SBWeZbk@=|kScbGNLRT9xj)-wrNw+LHHc?y zicO_{>E#V=GuH{Ds*6H=7HTDXdiK4=H)8a%h=>`uKs-K5i4eZ~0!)vsz=d=#Tg{Mz zqKTooZm~;P+OHt(X;uu-p(Bd_0&4JrE2}q$0SjP37_jgieb3@2jdMm+2W;N)f&PAM zS_Yk+7i|zL`$JT0o2Z{~%kt6O0;>B2v23nRS}m&!9J=`Tdq**K2`<2dE+vT-?(s0R4XNh`ox6=CK*Fb+QNLgJ4xB8 zvkMUUj|qbR`0(Nk85p`j!7J0~N8}lU%V&8Q1F*-6&ro3&YB3IJ#>FBa7)w}FZF^&L zk+O_oXyKp8Y84DwT@-fvKbrHP=krzXM1pxAV9>B}3~wo-r*j8|y%jmBqu`=3*0>P> zBb@{gmTI`V*FS6ao<=p#P*jt5<*bHPf5Tih<-Fbgz9H$Vr6S~!pPpbt zKuLveYUw+y%tpd30nR|F5$D@={9z$1PCv2KKTL0mMT{oqZ+8eqJUk?KQooiIbdCOIt^-OarsqMEqtNi+_}?)5iH^i zzmOLB=zZQ#>bvQyrLB|QSR%!1J`Rfr;(~Y3&Jtt&XXR>|nM~5O z5)s5v^@Nx`{I(+#`;zSz%LVU$*v4zRN$Gry7d3BI9cd=&6&^wI7RCqi($@(1HERSmpl_n8?a4jsb(LC#X`lI7W{Tq zEGL-eq4qeE@c;1YT!=epr2hp^?fQeKo=`=l*oSTTKwQ2ykj+O?dOTQp*H5~+U5qC~ zzd;uy*6|;6AJ#v_g*bKWyA#tzU^uMyaz4gcvV{%nTw_J81NY_T@aI8aSfe;n=hGZr z;eLtJMr~0pHiLmLyyP1nmOP9uLE>QdYc84fHq`}B+)_mtBmm=e#*fKUNiZh4GrF@x z)9U~9X)=LA33C>*?caLt1Uuc^ZvGNGb8m>sZVUqyZkO)!bxiKgh0~5eJC|0td7%VK zLVcGA6%_im_Ux&~MBvhf*MhS4D;)~6M1+->PRC;_QX3atH#@MWLgC$3yC}_7I4OS`h$N`oyW4X!=)$CJ6FE`O5Gm9C zC(Zf;gJZz{c>yt2_w%S>Zqgb9k-=f~zJ#(b)E-dXw&7ZG18Nk1f1u4F3;G}qg2#EP zzxx^ay)wc)A-097a{9jpm%Wuf*lwo7djW;NjGQmTw9$TNg%>`yb0lCetbPKE$Z1SO zNTnUYTCF6mexKA4PZf2WCgpzooV^K~E9Z0^;;L)&Y4g!VZcEWyQbP}-B+ZMy;O8W!QS&jWK*25%a!Mi&$~iyA`jm zjtIVt^gqiUGSY&9mtCA%KI#S5=>o5IBuqUhF^bgKkBJs%_>bpkhtga4+ghIzjsNPo zE`ujt(s}(fMg|k@n4Rs{GtsI6d_b>o?5*s^JX!6ZtTg5QWh%=3p#yH&!k3(q40R7> zZRhcd9;U}YcjtR9ssxAT+ROBe2VT(dv?-rA34~blw_H0~V0}*96eiw}d~UOG@G~ zYCRyoLsBTrdWTNvj1QwLmv#uS&-2DU-%B(ya~aiOKveyJva?>BsffGUMGUstCcJb) znE`FE*-k|RCT(ul)Nl)V>HiDtI&>Hoz)8urAc<=Y12QVliD>mvn!U^k&#H|K#%quh>Q;z7=^G&aobc8&wvX_%Bvpy2ig^ zu{{aDphwU7o?qRy?ppUR*K!tnX7=ov{l@cr9t28PK0v-4*N;To zgQ4T5!_F&#-g9lUN+_{>qO7h`XfBho#<2K_LtpWbe=;qmyUpX|vl32+@ej$ModvC3 z9(R(hyoX+NmdD9F%mnW(n$*UIX7ViQ*3Ni+q)K9}2pLCnR^N4J>=oOi

vIM~Tcweg@1kQaWSlNjZU92-BwAs~Bz@&c;X0@(Pbo`cQ5>dy4@PIip z?+9~Sa-~#}#=ohy1$j#9DU~dtL-*#DyXyrLPz+1x6yW=^KP0m<0=81>0UnF>4SXYi zO%d6OYN;7xl4(M$d&;nTEg!aJgv_i&91fyLOc-8klNPYoTN919^V)N^QB*xGEl_JH5y+nMkhsz3aqkg{mCWcdc4`o!@ope>DP z+b8zA0a}?A-xzL(oNVcTDY^vi%m?k+*tkwBPIRyiP-eBGj(`kA3nL#UEyVjU$ds0M zaLhZrdY^P=Fmz%645e01cmLs0!o}$^q0_|fqp@%XvYU-CHv&6;bOw*A&f;(6@-av5 zxw5N0c28cOu<(A`+p2CZL$i8d@xeOgW+xPaDe%k*`jE|YDC~5QG#lO5tH%1NgmQSNHKSF?TA#;-mh9M2U%0Hyuk z%V~vtJU?+!KTd+w$ZpimW^519pdfnW{?h1Z|ER7{xR1ATqHV{2rmCxFC4yjoD@Pwu8M6vyQh|^b$eZkTGvy;9%agyaz}}*tW~CQ=EhQ!K}cq`lar%XRHiH_W;uQ z+>;dZCo-Of1F^IOr#7^`{)|rOMtV2*7>aiO;?Z_Ld=j zY$Fma1LYKx4mZU~Nx3V&X3fL-kW5_iP&w^3w8HtP-@JrYC_Nk1;v6Rd3CUR&*&_|K zv`HtFO6T52mHu`=)(e@$;Ondtt_jn|1}r8}=i3gwS}789p5#T@%pG3={s#PGvuVq_ zdxNDOxwfbkP`=qSGjmt+&*RjoT56dq9-VK4t84c(dF)I>={7DZ=0yIu1X+4@13NuCFx9XGF>I;Y(ZzE}OQ?7Vt(ITE_pptcbv zK;2m6Ny1!8bf$jn-p8tBU(%LF!AexUu3ta|Iw4iQxoCTj-wOYDZP)$%MRaI)t=Bmz zO20O#xP=L_=bhBQ>1sv()YZ!L?VpRt{=b+&t`J=G5M%XRw>y?@zuFyXc98rHk879N zg)yP1MdJ%#{)bbT4k|{Cu}DC>8Z(mE^V<;_Y+<6i`ZFD(*$EydHH;06Ok`vywwb<_ zr`Fh)zT+}w7;fdCzdN@B2qa#x)fB|A5i?E$GNU^Y5EjT1pm#9Re9c3PUgOHW7qon7 zfA9m4y?RX%zjT`OR0^%0nH~*D3J=>@6Xx>4IN|jG#J!(QXE|Y)TAwsmo^gZMu1Z-=Bzq&z#&| zXThinW&j_at7|dLWV{Y#mx!TcvB^#v`?}XDP6Isd2^3vS`QjO-eVp4vbnFadD2H!( zf$e5V+B#Pdv#$TlvC_%r9#qZkGlom7!!4H*ts%o}Y|}Zd9eN0Rhd74@ln6@63p);x zAzE8~;{n?1wM>3RS6L|D-KYH}^YVSd5uDkyyf_nKb^Mi_h_(~N+vQ>Y56$kp#Q&h# z?GveDR7zXcG+lZIQLOBT4^aa!QXwCq--u!m!-UBCcy!VL0zrKp3~A&ZIzn=eAPX$w zz))DVhSBPD?=tH_CKuEx^Mt{iA^yk3 zpE9;F7v?z7fDR;mG`(2$G{l_NAJ-AH+aWd0icCk?ptpUT!?Yf~&2o^}H6Qj=<6+{I zUe)c<&hKk1yVa2`Jv-#T{c~fn2I5SDPxlbl*~fb{>wa}4Y)}0S&`U*~I!Bm34lqLldY%c&B0GgZECWpo#O=g;@_sc!S zoigb$)7n8pWDm0)4B~7Vy{YY&Jrl})Kls^Se2Hg7dOvitp-`;2dk5>1rwahL7FF80;TW7}fZ2TSC;1k^e4;>W;uu*fi@vI?#GR@P|d z>Eex|VCnJmP9)jpp!kc~4#?~)=&)5JALHj(=hq#97))Jo$Cslp!$a&f+>9^9Rn~@9g__UIx z15AwN5;r%+G9XKmLq7=z%bJM^pMDVzyd%dgh>23^A|q_VnNc`xX*&3AW#7hO*;t0B z?iRCFF3m+iT^t^gf+AzTBV>V*EKY4N_&7Mvomy=64s4(HEmJObZZx$ok*1C{hr;-s zc7)4?`z3b zV5KQpKh$NGvu5!Ex8VZrubZLiufTb+{>fbEdAi2OcY({At4q&@KORmWhk7MHdW5n* zTg`^ByNBB#&S2+Gy28qo8~)1liK%asm3WAwqD+|kXDjnPo?(dXw+GH4_^$8tRWN@?(oB9#IOeIt;pHBX~r#g-a zrP_wDp;sSwnGxS)#QjzleQ}0o>itk}Z?7c2&~w~u{m5gGm3D$rg+grT>8+x<&KMY6 zdcGtyL6-}x^cCymhg8QxmMD#pJ+oKZ7HWP>{-r#BYl@}PDDA?H56XRnf;T4zUGP3A zzzd1*WKnh@X+4WFtq0C%_fUt8K$-orUjLm8`N04&kB^KarfZia*r}d&H~oN>+ z(=L(2bGtU^X0t^WTbsykl}Dr=W`Za=rTap^=kymy_aAN;&lTdO;HKsl$k`_pt@d~<8Dpen+&T~YVeUIyB<;)k?)Q9&<0wg{F^`VeZ;{|}WT#?)yhNxWZlCW7h z&pK!DV0-J>+`hoGp_tE9qDFLRdG%Y&j=1rTIz4$)#2!Up8HU;>SS;LM4H)0Jl3D!9 z`-Wo>)T?xlYA&}r?I+T~ar+xNU92LGO_vda(#ksiFn%#{Edj883_loT;W_rji&_3y zZ4vVY@1!!7pxNy$!LTDf*I%Xex?SGxmycX@zena`DAYF>7#qVm*AR+vRuY+Lk{@5} z452tfM8bESBR4N3-NQFr@eK6ROQsdNz8h(3@`bdQRa(_3SM&1# zIFixJOPHWXwmL zlDWKvvDfpx^7eRYp^pZgf6ltq{ukt;{~jew?OMBeNWx|`hfxc%dOpiM7J3DkrA*Im z6Ldh=PUc^&6?k_%(f_T=mf`eH$WsVhh|ZF3_kmQijmY$owD`H{HzT-GSuK5!6tuV( zTf)1f#bU`34G5nB_-bY7nKfpI*WwOuuq#{wj@6`c0sQlKg7JqO^z))L{X;?<7csc% zd9X#)YEE$IBHy*Ls7E|E0if!p2L^Sm97~H=$E48>-2$$k3AejNZgAWCTjMK5xF1a( zWS8^8fk4}-RVoSTg;}nIE^wQ>X3@T1E!<7?fC|9eiC>HC%w8D7E6C=&=azIrRzyP|8e;xv)y>up6kn7f^v2R2u%Fp1guJDfUkve z4?2tp1MWDlS2aL@gjayo7&xaL;={d&y;j_MF3r9f{|3MQ8-TFYMJvt&)ttrGuoK?qM+?9%gF?AW9N>~xd1q+An*<-1qX+`!_gM};w9pe zLk|2-?@Y8St?Tz7w6{hvnG_v|u*8GG3;vkJJAQoF;ig z5u<$1v69_{3a*hj3OKbmV7vXqc=Zau9I%Q3*NNcB|IKJ1Vl?N()hl%D5(!^reN1QB zNnP{!q=Ov;LlqBAJAr@cZlxC?kPTfCRif>MU%vtSSR7*z3i9{~ufyNK9r5AWMO{th zu&m2zteaq;&u}VrpLi`S1b7edE8P?RF8Th=oAM8ixZuqy^-eVdeTWa+i6ICV^C(s? z)f1pv6#)*(b#hlVsRw>^(Wo)*4js@}>kLwg-62NGeYh3F=OEJ@2H~miCJyw{Qlyi^ zp5s0lJVy1DrLwf$E!B*e`-Ube(Ctq%5})v~dY2MXw173&O!z?$6=%|(qv<{CX!3Y6 zf!X)5xD-0QtrYz2S<5MC#;?(n=wIibl2klk`j&NDwy!04xrn-j0r+~y0Om3Ec4a8G;J;HG-pF3dRCWg?r@4e@RbZ( ztc9yW%U`(RbOJSBpT#z42X{fSVUNtW_q3}RVr-BhxvD}vBjf@*<4*{cWXYlB9PZ)$=S)l;XNSd< z7jSJ8`!_R&XG>yOTn9#u)%%2=T)Rj?#iztTe6`N3A7LgnbwTr0sq*TXirwtU)_c%=oprEY`m-G9!{ z&$aJ@@Y4DiMk<_hHawP>bf4|~;e%aF$eh-Nzcc?=)z{A_IPBF4j-fF`Cw2~Q>_OkR z7qyl=>q$%JAc% z8pA9#c9D)DV5q8pSgc#p{vakYAL)+0>HhgDIFK>Ft;{Rw+ z_tjzUxH*zgM*4?9KWF)ZG~~9_sUEZh?2DC@{J1N(Vq$4>0vao^3&354OLU=t;ZHj%N#v z?jqOIB)#Ynm(NnKnrD;+vuVrNG0aDv04RE~cfPPf(jDtwJ5=7*S`c(F>V1)#5J=%P%36Zyig}0r z#z@c&>RUy1P`)ElfsNc5O<)nby9JQ?FsnRKZtu@o?P%JFA!~F6jqq#(cgE?tC%A-E zJDRaLX;$J{T=+=CPgw5^)P5p51Zjyaet@=*;}+ntlgW2up$Ps3PhKoeLp)E}h?R5u zG!f~v4BiExV`Biv#!_BVkp~vYohoHx(3>&EAcC%ol2QkqPq=t||Hp&30m$!nZh5UP zX`H7NEdL?$jo?$p0Fef`uaf zXlG8XL}nP*P}=Jg_903=98d<;Ef9E3da>6#jfR6%1KkFsSFMd3{IwxK3oWL<8?4wm zZ)REZ=!EMc3+;4|{Gj~OXsNrmgLh?zm+R$~2YN6EqZRIfrV@gsSV7!;8G!0&63}sF$QRcn zdztWMEmj-Qf02|_n(yE!sqdsEq* zscBA~Yq*&SV>G16cLt8Z34#-xS8lLTk?!5e0R0QVIe1dxHhcIL$WS2!jP2aNzYC&u zzwj`LP*n7dLZB^G)%~7q?I!E__6NxlR}Ml9n%zGD&KfyOwScGFK$p925P$QiXM}X| z9d18`Rbaxd({)2HGVvz9Iz%i9#c$t>;>%$7A<6jw@$1$#ecA+frj4_N*QEDE@d02> z>E%IM&eLi5E^zOyb{NW)lyYpb$tMD(8_`2~-!^=8yqHM-5rtWr>!IsA@PU@Zj|N$L zLelp@o6<)t@a_jydevt2}`MtSAkTGxuPzbd9I^ zuxW5!2&ERcR%k}1MF|Zj$y$8wtXZ0+glJ@%!!GgRdcDJa7k2h>i?P_$a@gDA>_-ys zl#7g3*IXNI)>FNQP<<=B?yx?|T*VO)&@M_JzE{G*g7 z9aq+M9NW=}GF<+EsXTk6Pq5F{8x)!;;?zvNO^^dEy622ztFS&$4`O`Q`C!E(`Z4)r z6r9L^@}yru$cc`0lB*6d+w_RNmkZm2gO_^90lz(nb?VVOGkkW5PcP@#QXAFC&HY_*mp!&`oCP@%EHW#B_``}~Q-u_QQrLUD zgf+cATdaslRf_~SzGTVVF5$u!&)E7Z_pa+uEsFt@qmWKWw~%g3bN_i`t%NKmASP6( zrKURm(7?*_p-yCMI(cj)-yx?}{~(T&^RC;h)_clSaxmOGl0{nsZID;!gV*3D0^kL2Bh~A-7k+IUuokS-ntEP#Vmq(jU^!*phWI zVp{0lOUGNL82`<9f5WRvD=X@O8}FpDf8?g7I1Bg9$=}tmOb4O_ffas8T@a2 z23>#p41E6Z8Nm8)$i`nY0st`zZ@-PL&bJza%cgbNT3=*TOk0KW+R1b6g$7!|7Q?`? zhCJ!mYrSA=R|;0P%jecp;|STy2qSUBs0p{E$HHaeTr1KQ-DTzNQ<2lR;mB&u8G|{R zK6Vem^}q#8L65i478}*@w$9n*7J+ElH*6z-4=ubIM4n_f<8}79>?kT@(6jegd%Z5? zP;1Oh@A75Tmf0F3;85T}9Ew1JqpYWxWlY;g{5V<>kEZcB%JattYa1UXkZYon)<>_8 z$@0q6_1JP!&P+Yzp8&=}WA&nO+4P3iVF_Z9l$?X6Xlu32*)((j(6wq#bmYOI$G9f` zL1(bH*JHq(()nWFhWz2#7?Za7LEDH2>S09P1BoB{L*I6Jl7+S(Vj%(82!LVzmycca z#Ma%D&&O9|aOiF9Y5M}JYwXke6RT8C004l&Uc!>U;5D3Iy-q4i+)4HeU+vyb8f##x z`~QLKV0`-jmFqxkeqP{V0`Y(EIv~u5=WqVWpAK5K%Gi0GdS5#1inC*zhq9fa-Y+0y zuSsPmz*CfsIln`&a{2=2nscqdW<%uX56l?tt`>US9b9hQ=WEBJ_Kmiw0v=yv11TwJ z5I`0Zx>KZkI$SN-o+qC4B7Z$dBMx+Xu$zm|76^}(8)RYt;bFz*w4de~u#YTB`MrAE zaEC{w+^{lwFngE>=*dF3Pxm7L<)Scp>zaDw8@$QUhPSsaT&Sg}!B^oYlSNKDj$LTU zgxqOXiIx(N_Ra2wt>XyViHM!Jh*G?C{8$Xnaw;CBEgpf;3xh{h#d;xfN8y(4k7CWb zpU&E7%-2~XomK_!EynttLUXpFjk52N2!I{)E$bWnJT-^mQ=YI}3Fa|1+=4^APf z_1*}txGlO|caz&rtKSr1CfKCK4CbDCy~MHLl90vk8Ep%Am#jA zJdt<#E6s#n86An(M5tcF3ST=FBHovIncFXw-Q99T9T-0^^jqwQ>6pW*Dbjr+AF=_J z3b>5}*IQmX{IcK<>4(tL^?2Y>L--hbkqqLDUqHs>fzkejfq>)uuf-W7rE*l7|4s~< z2~pKZ((?@046pugq~pajRGi&fI#D(8>6FD7``uvB&bz^a&grKUAosqy6Lb)0Tj@G<8fg4-+CNSlw5YEmHi)2 z=s`LF6XwK?Gwlcg&btq6jPs4MWqGHKL!wCe;~C3s?H2yJ zYgLQjp!G$IXX&QCf#ydQCSU0k5oMM6!z^C&rxpoL4YU~{_ggWLVV!pWpbQP>Jin^2 z5o+o{->RUpuOR%8+K05nZB)|bc=Jn0^r&&FZ+j{^s;Ln%4+~Ys5cwz`g{%w zl%kg@_^-5ZHjjEsVa8mIn0J=IQnLti{R`l zEfeRDU3BsVdUL7376b@8dBSPCEV0qi9&62+p;>8~;^)G%gI6DQM?J!)oG7$f`6DQw ze^r&WpMtUE4Oi=L$Ruj$%28Wn=I)-f0nG+Gz$OtFO~+z%dw~zH&cJ~5X?{ZC89=ox48Ji_`v7zfaWaTlw%l#+w7p%Vu-+axxk7yCd z%-p0VTtWuAa^Ri}j-Q^4ZaVn`pyH&?X9;?}^qM^^rl>h#+TpWFmTR_O*{<@jXC2-^ zmkY4s0x;#h;y%{dq7kLl`hsQjL<5EpZ~JE=1z*%9PC>+$v)W;rQ~RYam{(|DLi-J4 z2C>utzaTQgr6Itl_05c5zn^Uxccqjwz2+)Wl+HV?s{v-;x3=~pLz!k!CzzLD2OD?c zVX6b`4lh4m5X>!B-foY}Q&-2YU`F@T?gGi8fFnFf#Y8JKBHi^;>Lme5pivav>62?# zxrE5<+4(`_U4SVJDK)S9g2idlZm9jr*D5QHpfT)?NAO{Jm<9cu9$-He`{ILZ_77Hh z?g83Ig@Xb!71GH&2UOOJ1A@b$R_J7(VCl9Oyk@ouuF>%4d`^IFT)U@t9d`0y&qvpf zB1j*OT`yR=t|@O0|H3@lHp&oWt256z+h(~gPaJlGN`@qmiZ~X)d$u`0364U9i4T4P z9A^%=VWW1RY1Kbmmz-yVg(}k=WVV{A{>t>yo2e~ErZX4x-}0-jpJw+8&=mj5-e4`1aQ3 z44rRV&&*-$aWC`hu766pmZ;V0IUFo#_g{0KK7EK446n#zv&m-p{zPlc%TMu0i-UK< z9yBs@LJ}mnXH#kQYx#wouSZgf=n#hu_gyhI2XJhW9jME&NCUvpK>uQ_kO@$Yb`3l;M19G$u16w;p*6=0j zqm8$lx3d9KsxhQz>AV1KiVVT`WPM)Sky-Qo0g795E_!D1LvfR?pH~Rm^E8( zYH~~Vpf}qVV7Oq|w|rgOuUKZ6r*1V21zVA({)En4gVO8i{{*GaYyKUS9zDh3Vr24} zorB=u;oSm)g&&-EM9u(!is`*b@5`gNDFk20q758On59QNLd6q^K%M$V7c|^Vh0TK6 zQV6rc#uaed`p<&^^b3EDs>|;g-?KEY?JV`}#TBj?_ zJ||2-5IlQz+TM$)3(TyuGwLq}-Vk^4c0D!eN1!_JT?I0w@Zf`Fp2h(f!YWG{En zm7(zf5wNXv?QuB9jfofpceZ(C`qh;(JK<^_ElO~k4d3b!6Vld=rplIn7%I2BM?|Nw z=Jv=De|w&*rMp^)5c+%6%pMbQO^a$gd5nR+UUBt}@I$;fD(w-IJS}7cs?oOnnd@EF zP^Zp>>C7de>1ur7Z>SI#Kin7O8{rmC&)1_60B`dKxK ze*VI*53dI9oO}AWjM}G}?1?xEoe|t%mdhtUap+y)vTx!x5D71w5Mz9ADOvZtRfeek zp0*>J_a+?Q!oayi1@87;j-TW`6}gt)(y6?5XxxoEyruER`dO7FwK+BFlJkK!0cvN%{fCTpj6`$C>EN1Sc880HayX^TF}>$JAJj*)`zqdcm&Y73HgSS1J0>`v9ECX7y=p?*C?I=PNE@x-Zf|8` z<{3Pq57V#Jy%ruQpc~x+XM|Ag7`(u;-s-?XS)-^RD{5ZP=h5E((fo{-zMIHaA~V4o zJl8nu-7jEv^kZgs>%sU38je02k~siniS<^*Rqt(T1cl7)Amzp{ys*F@vXo=6PnorZ z6Xl0QyVWZqJ2w6qrZY^$+7|^~zFi5Cp6u3-t|sHipuPJqCr}F->UjF*8e~JrfnYI_ z4JLIaABg?XI2cb3!?oW4GGBfy1*0jI)dV_i+F@WkE<)gle-~9ZI1&^qIvjgetF{^b z7jAv(p$tp7GDUyg9js7nB50^6VuLcv>LNxTvFX9x(~mQ>{=uQ1C7fX0kQ#J;kDSH4xhDqnN0Nyq<&=}9glLs=0{o4#Atw*e>9 zu#-IqOd)z1xp?YgX?B_$yd z!XZ@4x;(#O`0>eO~J}$3Iri7dL3AXOSr0gF+qVF2^Y>|wcw0> zcu7p6cK_qY5sQx#hxu^hv6ugE<8f#9M<8`j-fI`0I;aJ(Yr1#!F0HjrHe89-o}kj+ z6!xXMRaUVj0ayv?pllx#FmJkNubGn6}(o0a0oM z+97Zwfr%irr=X_zlSFS|6G%xXhV=bnyH~R{wclnnP=Rn;0o60;8@zh1$G(Sf#IY%^ zVuX;e$^RN7pgj4X)ibmr&%7o}2U~|;%;J!@Ef5Uu9?08<1Y_Htb;?hX+#6g3!pWN5 znYi@d>`ds2Y|8wdaCuPn#>=}c z*eQeq!>qg1B-u9lhgl`0fA2*!(3;PBcc;rD;`PaeUncXuxxHx*h}+>-4yX0^N_Q3H z&3jxrV)|2yymUbO`jX1NK6CoC0vA7SVW6K1W>^Ljfu)C>gP|W?<*-guZ0lwUh*bE4 zr$-2KoF6Gydc6iFNy#kSh@%fV#*KprT)uzOH1-;`?RA#2gO~1muUhd79-WvnFbISD zo^KMZY%S%E{;t(W9-5VIvgtb*#3Y*10f@3^%p{hpdG9cc0hgiKCVfc!(vA^ZkEQ{$B;SUnZAx^+xHhkOe;8O7nyhm zd(O9Btfiw#jo7Hs9C^SCeY+3vLf>T7Aw3&2b&(7Ow32MpPB%a@(9vhx_$8b@a{;jl z!VR;m7;Q8jHbu%nz_l@<-L4Z3NM*p7^X`Ww0(1T7Jz^UMlk#umoyFKOsGi|$DJ#ba z$NW$MjcftWejC6daG}bz^hr=4x#SBkNO)Hq#@JA}zh~>ZVyAX!b7%6Aya?|^U?YY* zP~oqJS>vRF1qb|26PQcKHpkk>zI2Dl)_7{2+}=3^cYoM0Q(G&S_9Q_LEUQ?N^muL( z<1614J8?7G2814CWSXz$>rjp+(dgy1bF;RNIc+!b&2ab;mlghbw{VuRojuVctJLXr%x`llnb0#tumxQ#rWFS0(?#$xwopMxwrO^+P^xOkA;I z3FD^ZUB2>mP1o3DUvk4>`JAmbu{WpAw_HKLvK7PcA7t9bChxRpy2Da{GWO*1E40qR zPVGI@9E$(0lCSL^m2V?o=YSWlnSf=DMM?2CiM!R&O{|;n`S7K#agSW+?Z6bVf?dT9 zKJ8-+hXp+ssyv5U&{K0)k&K-KEv}bNL-19oMqcq9(J$GpL+PWb4CWW!{HXW&rPUU* z2OiTkxv{L?yYu^y!%>Y`JKM3!NTSFhZ;e_`H-}d9hOwT+0c}a|>$HCk5ZqE2cL}T;r zbl?^${dxP8v;<#v4D|SkFUpt(GsnHqoICj8zN=>6nq438_=t?Lb|A*~55 zJ@Q9x5RP?4u1R(6n}rj72Zy8$*qZ3>7oO1gRauqUr$L3eQGQU&R7?oGR|9?WT(yLYumpLM(1FlCdHG*N7y0PNyP(14iE_k$7(; zsU=Y5yvO89B9vb}x7rcRtVr)>mY3}|xf=kaor?_mUIju44gJKy+c$=qOd;zFD)|23 zcb!xYtMosN+hoN{)AM+*vEJEgCoMvCof>YjDm;5PEugG>eUL6;pHX4T8030CRfkw% ze;1fTyy)d8P%8?&qV+&A@lnJ8Fubq3VTL2f7E-+!-oIKA5oSFG4FLCyK6?La)O~7` zV+Ox1TL{dCb{HA%>m0&CrJg@-@b2vgJ&#(a)`jzRwkh1Iq-Vi8&|7y7R=nF%ZK!~r zq?z%M1Twb~)d7B%+EIm6%WpZ0nW`UhFT1lXmL;{vZ97rrcv%-d$G+g{k|k2r>5fKtwo$?WQYyhT)n+{PEO_SH{lwE@nVqNuX#2 zX8lCoOdWgJRI;CRe0%G)XMSZXp|FB~j+%MYdC8AXjBEOVVwpP?fUiRo@tEAO`>2;Q z=!Pg2S{ps+rjS?3RxGiXenjbV6FBMo42X){Z!|I*H}#rZ`Q^6P1o9>q_+d{sFG{CO z36i7Urz6pK$~Jd%9Dcz;{!cSUCWK0rqPPhCVvk%P6#60Dy70-Z2sEC$^$!n;ut`sD zbn+7Eff&B87^hLrt+%#Re6!Nx=j02T#a&L+@gx#uS?!MWv1t$jt3KaGesG{oVp8Jb zKV7ZdFV>Qb3_lGkLqtmBSoY zM~j<94$p``Und#w(ofi2rtYdFXTvx_;l7H|HdWfca)&FLUjcTSuII>y*K=f^!N7hE z!t*1cxIdwcw@Kx)XCMEtX8u;F!B0}u2E_efc~zrGJCcQbxj)wpueJqi=FIrPV5j>> zI@~5gt}1yi6N~C;C|!wx94anDP_E8mEK|Dxdp6@E7IXVAsAn~r^k#hPZe*)9xvBul ze3*v!?}#0D^CIk#Ux*z5d{1WYX3*7JK|Y)0aDO@LRd3tjIHv}0$i_-(%{=H>2*mDk zsyzTb$z~`H3?TLQ_9DKjc`?^*??%)OPq}Vr+v=${r)Qu)Y>X}Q!1)uvBO@%WD_OR> zbu|p;aLLM)P`KIS17t`3P?VZoWCk48ZBdnH_vk_E?-Xatb%89HWF#RX(XKGtTfU7I zTLHUH3NnP@eNT%6=?H;RQj&KwR{4j&)l%w*{O@I@`niK&x@~QLWNs)Eh<>sALMSdd zrYCL9^PN?(Vd#0yEL-A5{Uz|MClUizU9Y4s2a36dM+3lQgV2qYNjXDBGHJB^gn;zh zz-G{e@C1kfp~0#ylq?TXSuc}M751F__A{KazU-zN5O$rdlA8sG7mD_voy+uxwhk3q z0At`O9K)dZyEn61LnBueNWd-gPS{-H20GaCYyeS>S(XZP($N?1%g+#ktVKn5_@YWF zBlGf`jQeRP5-}lLN-@X8f|$!VuVp{^Bhmf@uvm>BD``a6hI99-hr~TBzaVcxL}Mz!4M{IiAWecOQ$AIF=$i2J zw~$IHB?jw!@kZnM)_{{ZugIpR>82becd z9O%~jy^B?dxHmu%{>xSD^6FlJZyG$6r;z0Mw%u#eXqp!Fdp6dK-++MBIRhNl-*_iT1uDtgfFd_85LJwTfHwFvl`5yGsBFuSEf}(6N6Jko1D0JqFO2&RoVhtVMObpDy#m9~D!=_-@ec zjKG+=77!~uuzid_<-h|xAT6adyXd3^yZHC#HGBR6%&LU%f4qV=y@CC3BnACbH7mr4t&?(ykN^drK`4eK$+Uh6^ zj*-UOc?vE!n`Xyi>`HXJfdaT*KYc*tkRO(n%9OED6c<=d{)q9xnj|ho=au;U>%9hc z&vFRNpVKz0@n)q5=G*`oTw?B=Ue%^s$|8+EzIdLA+KP4xJP{UFrMKXt!PT<r z_A)b3)Kl|rV1xh<&;tJGBzWEJe6boWlL!#L4Een4Q7tXz63&Jr{EM@xcXZ!M9&XtL z=qA0t#Sf1`(2;#HhXN#zHf63nHF$4M_6+zxSV}?@{6`%_8%Q4AYBOnj3_otbWK-(B zY!lM8v8Ol%<34yM-8E+=)jI@$*&$sIkg{tk-ifGF>zfgR8b#x1{ZYfVi3YS^*Ra)x zE5PA@&;oUAgi!qxY}E3|h(xfPgG+Q9JVive>I_( zP^w-B|21>nY=c~LhkiOofcONm86EHMBf%mAC|V8ttLycFOc(I0&5xGH;=JlUpEJV) zXFBr}_VgIRoFU*sF@2t1)~1`Vo8;=KeTsJ#xPT~XG*|twSE0CX*7^|08WM?U{udj} z&#>|P9^PImq;`1S@rkcrKY@R50)i4&j)|&b$gP%%)$jy^ozc=jYj+JV9JIZz4tDP$ zJB;tMQ5K~(l1GzA1Ikvc4Z+n_(jo#Zn;vY;imk1m1jIHRDSL!i37;asuhWv8x^~^e zUjO4X)gC~XgvT1f2KNVqa;O#a4+#!3xjwIq4^Kx5 zA!EtA5kFO)Q#s@{Wzc#rja8(kT)@x_q+nD49m|V*@T7sS6C4(=kO_mpV!Bp-ZN|vVJ&Qfo*|{T}iS zW>xpnZGn&2XDaFaBK)WG^GZ>(knOD<$Cp3Sy#NLob#K@ZghrhDXw4ASaY}`eQOLsR z>1aJ087iyCo|Us2gt>ZLmxmFV_`It9b_*t?2__l-@^04eb}H^f;JqmmuSlf#U%9!f zD7d=+ut{h%lW&Tff2bFit?PrD14sTN|QTEeg&) zRL`zL^U%~4^46gluywI<0`>1*DRgyF?I7CV;$k|HM^|3`Fms604E>bP`-ftZ?|>m) zh_&)m;5W!)`qqq}316cDR^9D&jaGR;f^Yx1mQ)*E@Uv50_fb~b%B|)Ou;SY)7BE=R zXYsDjWr`d$w-}!3^$?W7QC7_*m1e1_@F8ZSiaNe#`^DXJc)vHP=X~^Hi>ac}O<&|h z>GCh3k?Tni=j&&dDeOiKbZ%EdBeGK2cI`K86LS6KN&RdDTOu?HZvs+8h(DVUUy)9* z4~fDF0>+@a;uh?iW{i(J+{%#~QYL)VvS04aO(-PzevaQuu1{sKp>PI?&o1)8gMgva z8Uuw5zhGc23LA&U8V-x!u9qBu3{QyqS`h|BMvCLVPp% zMAl=jyr;xcQm@B#+FlQGmc0}2YG6GjJ?<4EzTIUasxb$t3AZ0PqjJvGL@d21ZqDMw z`b7;GE@xsCY6mN{PCq%7^$1+BU-(g)E)tb7km`URK4g?0Li2ZZ6o5i|MxeYz@YAKB zj=XS>d9Qb3kJY;P*tbJC0+;rCOWB2VWyk3Y!8KsXGRX1Q)+v@G->;E*|5BH3<6i46 zAYQ#e!yjr%vp02PrNW@@X6D~ZLz zU9LtCgfb5@ZiUo1@9%Tfvms~|Q4jJRr}1RTYnt^Rvx&`BGd+4f6((i7gL|_|0a2|y zCSLySDAG*M0@unlI{%`35Q+Ix+3*5Z?BVC}0@#nTF4;Q{|`ft+( zXxXYra-TO`gbjZ5_`5lQ%7Nh(&=CW%jj}uI2r_gc6Y9XqsZmhC;!zl3E-3&72t_Vj zZp6CMKPiRISkKZ{1@JjO;-f>a|RCQ)gDqUjOc)fPs%e{EcVdSs6t? znuqHP`RJbEa%vgdKZwlf4A1Ut9gZTQB)khqxhAJd$ke0JP0OH7+~i^Dw&fy)j3|jt zv5TWe?v~%XL&xg5JW5nTGiYAIJ9gp|_louKPj?9I#yXd|~e;WZb(#RHe zLlG(Q>&dvtY?s*qkF@#3fw;e95glptqsha}-2%b64J>q-Z{)mIdo!^qW{tMN6t~pg z1f;`X#RkZU;2~}JB6JCu%3L53G800r7l&dD`fCjhD4I7p*eE&^aq0Lwxg=_>!3xES zZP&fI>(pI;gihGCM07aZN9;*whk-| zGg$PzgXUM_EKlye=Z!w%Y}#+*B>F90~*HBJKb=k1ZoaVBX|M85~mBSfDAnhC={}y4mhglbkMYvF`rg7;j zxNL4c7DxR)AL@Vm++#_yK@|zu%Wrv?muyrw7Qoj**}K&K_Y3SvQqP}sRrEJT@Uv~> z(o!~_eVZ0fDZ8gLu?u1AHgP-W@xDJ*c7<2?s&0~ge)=|GP(O05^BbWTb)4;t_~l6A z>)ac(7~?6U`wP!FU!#O6qK|8BlO1G|O$xNF*Rqy4fjKb_%#*nclCgF53UvokUvANP zf>2j}ya{wl0WZ|Qw{M5RgM7akJ;tBjQdHBnf-&vRMv}cMYuUg~zRDJu9R&AD?@I(9 z_EM(8sJg>|wG6?5sN9=bk-EpcZepmX-?spfNcM{-ud!DW8ZilhRGBH0h{|l@&?AdB zXr~VJpwaOvQ9yo~#{E8y&O~7KaTJ!Uz-`;cwO8Le6}UE}lsZc>AV>Xi^ZfIaL9c)$ z!`=d53T{tx$NMUA&F_j$%l%TT;-~J5Y>o6`AkAHi^ZA78+xtqHulA5-)5c>GP1)W% z%;L zu^Cm3g<@>y4jGn_tnjcpDz4W!gFhBBVWr0*k8#JJtd)iq1gx$Za+nkUZe{sXk_IDv zf%zmx!ITd=!^em@)MMzN;~L+3BJ(kN7SP1vY+d)Mw;|WMvVUn|Z(rx?0&|rhJHWf$ zEbdB8c}E!D@DOnK6D$9ud`-XpMfr+`!xS<-Ixb3(oTaB{#!K@e67ccDLrD`JgXdDUUDdER zEmZU&KDt7my=P-vEQ_yFE6`Livk9;wO+kt`X_yDLH70I;EexWGSNS0RbT0B4z;zpf zJq*8OpLK?{rkqNwSnp*_OuP6~ETbgJ83R|ZCXGj>=>8f19^*cSYtm=q?*?u5zUDw$ zrA@OAe#vz9>0;<~BLBDPRmUK_`UkgCCz^x&*fWL^P5mKN0W1O6Q|lv;zy$vVIuVuh z*4nv*r0>OeeLl9Gn!!yCLdX3|g@2n|l_i;H04t!}GpnU)MAu12G^JtnrL!WpZVZsh zCFB~ge_VKsF@D44HTnma7ZGs`aCw1ffK%LWdhz}`H+{@EXw6$-HjWO~rSuds($D1w z23OqYY8PtzL(*UWnl^RxyWHe+5UhoCCi@Hj7p)l?S&BzM1h(qJe>f;QB(lP=@<3Q6 z`zi>2fN{gz7|LexpQs}&2`v6#|qbmO}dI(Tj1(@3Zp=|dxBt($xV3Ui0-k;#FOvXsX!#z#fLQin5S4H>NjZ5-ci2hWCphM7jmiyGL z?t0b|fJ&;gsGIz}I+OPq1DNgtXn+W6H3fUao6zT*tdwu8)r3(a_RiT-J&ymTFHb!K zdJOWOWk4t*@RL2}Kjh_o!WL&jxT_CSZh)rxm8x}XCh;mvIU7B#Qq;d_R)3vU3>?wV zmI9*);SIEcgDsM$VhRBy=&8+G@s!SkoncMJ2>G4Xf27KiQBA&L z5NZJ7EBqtK_lplKQH;$hx8_tZH-xJ|<*o08>)*!0_~*ow8$CES=%0GYw~wjm_?&nkURDFRQd2wCy1V@=+Cbp2@QFfl=>JTgC}Q zfB4)Eybd#5_tK#RJp2Sad~j8lbn!{*y>wStGt_lY68WGo_lSe9E+Mi;Oo4d0OG?BS0~{?l{K(}65^(>z(Z zM=F_*bN0J|ZUyFUb-~Gcgnj@#VOg=@bZhxtSUVY@Bd7R$5xjj+(l7?cbOjg$!^?p{ zhvN`_d{)5T8|ZvG8kWiE^kmB9?3iiA$b$QWYz#gi2k#9CVIFS-H)Rt~K#kh3fDz0n z*%;~@oj4;7a8aPd`r)Ru8hF!Vko#K@#`#;_l)i}q7_VRTm;)54?^Z*4eBPSmD3Zs) z4(ymp!>N0EZ9^=qr@udoN_XUEkU0$3J`sfyP{l=c%P1mk!h(-#qhw!PY47M$u&?b@ zJeUrWh%|e!bifZDO=0`YSk(<*7mHSbdnp1X0>IUFAjKI>QJ+fc%y;+I_%Cirk(sZ2q{$hYt5vCThcWFqSS z)*_&th6QDsSx?@RR04=|1HSf4b5>^pR{4ne-zUbVB3Oo^)~j zE`Xooy;?$eq|HYCa$?x(m!$KxW5pZo{K!vd?h}|8TUJ|lU%N7DMmsr{zTC+eNtaYy zT$%G5+3Scg(ol!bNeOM+-CpY^Fb201;YRLwa5;d*!S|)Z^Ji zcqQI+Bn^5xVcuGRm;`Mr`}$o_MU*vS+NL|BBzD+QbS@gRhc$K5{u(@I@7UDexPbyX z2W95g#{WexP~m$h&q>mR0nM#}j&bAT9PUsDQ}w)P(z!eR@`heekEfifoK$$#vCDsH zDmh=1fPLRS06`JG<%FRh!`*@9zc81ht|?)4t3;==vg*Dd%7wKf(t7$x;v=K3E~r#? zUJDS7<1%;-S!aB-m(=a8#R$8<`Aw*tfirVYZcq!>-npsnoD)sSpR+$Qj3P(nuy!N~ z$tc+|U}nZS-?RU0XT@`}+>c5d0?wuEp%y1I`I+o(;{~M5$y&N@2BYO3L}omcwc_%9 zG2Z9%_EeE(?}^sd%(xg-r?#e%y?oT%iI>LG(*vZ7F~xd+3;VcYf#U?a=W!OzPn?O| zKcs(n^GQ768SE+}9s>Sq;a?|DR>iDn+#oV?dW1v;U!z;gHyg)4u% zs_|v<);7XbfL6LXe;f2vh5{!-v*B3qk{UPjL)oSD++ccdMh=uTz{eH`?ZJbm;g+StN7nFZj%S+L?OKHCq={Wq_qt9$x zex}P21ZsHG^0QbQT|muVR9?u=$(yVvQouqhYU?b^I)NhWl~%v}%e9~!Kj70&(46yPMoQF+7ji8r@rSWx zMJOYl$%eod^ZLE1GHWXS=iiNF<3ipCP}%XvSNF%2>VO5tnh$Sj)||DJ)7fSl!fqeY zIj}I1pI3`U;Y_exGS9tj(<$B*7dbMyY*ngEud&uDEj8oSAig+*eYH*WC@n46)Qprc zg1#pbJQ5gLD@cyru$GEU`}vX?K^`1Wvb!8tU^6i)c66`WNjKzhbCC9B?()RrlH#j* zrq-82J*9!MZol$ZF%a%abW!^`rqzDj)L6BT_WKmF!@@#<80q7V>@KtU_j&d5_fLN% z#cTym4p(GP33B2|E5rM(%X=;yFgW-~Wb;k?@S}a2Oq-*P=PcW9L)!Gy(gTLqv4${| z41EK=qdeU%Z56wOknqK{Dw6URvuCqzCw;hm7rP1t4I^LIme;sizeVs}GiW~7R^%01%?{jXs3<;P+({9J>7sMr^T*vQ;i0h|3 zuLKSjko0`!7-Z`tX4vU$`?RW?ks5tmUs%>R{OrlyZd=JJ;&<^nHS(6-vy|Y1HQ5E& zr^K)4CiVM-zov8ET^=s5`l$$8gsdapsC*}3MckErw>$0=rw5LTU!$$?h#XOI<5&tv z4RSZ=e}gQw&@B#d%$WvzYBYCC>U=<@Bdjr@QeSdj&%r4%osYt$uT|qiRDVJj-*SmK zMj)n-mY#rYjYB9aq2^^pxcyif_r8j24fKinm8Fl#+m}q_imOeQ%q7OC%C*?bgYrpW zpvSMzowC6-E5sjgRf}9+NZHR#d#2Bo{tWrF2kq#gz>TqRm#~&U2=93N%^sVtj$URU z!AGAqv>J1&R=x2hA9&Xrz!!`JMcW~b`8P1zeuDvg!BHsDI{;sR3E&H;qO_YIX_-EK zSuM&6;0t<-s8Qm=-!ui;aJjf`VprC zm{Y5OXuNX5#Hp)$3ki0YYuqBsWE5AXnyv6+O-qibQJ^;9esr{qjBDxwFU-9E@Kb70 zD{}H)4vAi41=Vh&9B3+5co7hdn**Y8`CYl@FXx(AFTecK?$7hp$UglMZva86M5k=$ z>G2G>VM1~r5rzue%U#lx-n%g7sH=h6_}1fu<`a`PDg4MV-Wj6Z?=mlGZ{VO#Z!|8W zT>-aAQ{}@HPN4lg7`JD-oD73Qwgvxe`yJ4ve`p^Fo?6cZ8<=$Sjg_b6L-C_J0(9RU zzr|Qm=L1(+E@kcI?EhK zyw>u;NRArik`o>ZJ2-D!_OklTi+4<9ZN4tj(@@bQOL0pgV;LrYxu;W2Q<~-8y0o>E zhiIYWRShTqf&7jp;@4RWt$LutZlw?^{Yin9gUK*v1%Pe6mdNm7DA8buS+6@cX*?k> zaT#K$Q_)40U1G{H_|zT?4H|}wrc$;Rpjk50$%3on)Axb#z~f#q^8mi!^t{nPP3w%h zRvdNWyzV*g+&?L^O3V$psHCVL3`uaDSs6Bo?NyoqjpDmS`|>DWxwt)Bop%Z{0nz+S zWSWb^W=2BcNBsH*Qgc8EZ0er!L0aL`hc=%DN(ez(@LAVWoFThZevKhzjrj}iQ!T~5aEzkVwRZjf; z_&Tj)D4#MJs>a%;8p4{~lVE2!sQDaHVV^HYpwit5LO?a%#=fBhlr`e&O9=!mxB&wLA}Fy zza*nvw*2_%OF>5FlrTX4{Z6!Tf43#%(n4@5n;R+c3cweT3})bCU*ikBsENL8=mbBX z9bK{kZ^Q1@_#-mzhO>}O{CUC?em2D%)G`G2^oY$;rI6Sq1-E)1z(74oJaw5}$(EN5 zP?Nk#9>YV*vYzi->QMVSfR#p0i6M11@c9RUvXg~P;x92gotZuTae29lDQeHQ;Iy=W#obMABxU6ABQ9X}6wIRRckG)~C5?LNyH3)5^A zT&WVuHifmenMoX|OJQ0$4|;1tzj-Fi!HH*f4JuB53iF1gURJvnSVE#?FxISRi-^sP zj#fo?_Gw~*taD5j^0Vs+&)I1cYhH_lUaqHJTJQd9Rc+&cJlIo~Aj%_EfeB_rtNK8JrHPAX?@3G+>ZiK3Ft zWO+LH1a|eZx@+r_V;-H!7Ncf$kEk$*l>65$dToZ_A9LR*3)_^`l`=ziisf7&zfDA| zvAynlpq654@F6|5i5f=f`@JoXTEv2*D4iCZcbF&PzC1!br@iNcRJ*=(gX^)1WO0I5 zAcfcdWBE}vP6%g0qpb%k{yTazjybE-gnkXVQCL*+?SgZ8#{Mx<))W+oIN0Wah+U=g^cX(;T0d$+R@s^9vztYp>XbJ%3MWf z0(*Qixj&0~y0TvVKisJ2#}g&FSoA?u z=kN|!#M`jmR;TaVlfQS$B46Q*Dmg7Or;T#PI`CUsFdEI9?lp%nWg5yy#jt-|%P=UY z*>+-k%deBIMzjntB$O$!X$LAvEk%FreP-r(F-AH3q0`mr`ik)3QVTHybE#TOo{@v%AtlYjFXdl0;id z61}%QR?vx*S+bCoGT**!TM_m3+THoUkc6Yyax_#JJ?fCvCEE3)c;bMlMUluaa`9QM zgY(70xeHs>`!D>Zv)@7b*0crR@*1^4!18X!nzLOPF-plXY#ag1F!< zBD=(w@jqmnrS&R$U>f|fkP5jx_xmHuC$+SP>#vf7XFquk#jZ9EQ7Y2hO(*VtMe*?r z@h2Ut?IHuzxO_ApKZEo$e1U%joN1@eMaPtXwiZ6wfeqWSbKoGvR6Qo=N9OG344;ZL zvFIoRb{_YBqV>z=CP~7L*q-h@=9TiFLm=^PB1blbvZgjphQi-f*vVCG!=VT!6Ia1s z$gabDKJcFViK#hFSNQYH1o|k~?YT9W+DuV;){;x48l;6j-*qIm_zS__Mj~U>?DU<2 z>cgHq%@s?Up-B(iRH1Dz={)ybls`v?(Tdh_*gyy6W4$86J0+89T8|E6&6=0))>m$k zal8gpJyoj9E;_ky%A{*Nb&<8REp)xJGj~%jU~kDM2Ehy(cl2Rt0(s$lN|5iE*qsKs zg;e7}!Hj%7_zMt$rjoJa3C0*VUy@WvwR2C`2}muF>+&%Jn?FT#Xf)!50am!sz8TI>|E=If$5V^_w(tF8| zTgjD8)&x#$n-8PJ>O}=784*pGf=eIP23^i6ipq`)&XP7&izj&n`57YDf6~qxi9p=% zXicjRGp+*kAA9T8ZRt?nF@#P3Vs=l+m$~b21Apo@ zo{zm&gp4~nIIk$UbwrZhLK7FY@F&iu`JkDcepB0bHb)X9BrARgImnb(5)ajKWpu|-zd}zi_MB7c2-B`p$DE{!**r;wDd)u!2rG;fF=~)lQHDCdJ+tmR zVaSb**^sf5e5mhBEanuR*m=jipW7hSInM|y_blm&xXWfhSoAGgFp=MdwTBn%3D{5lu_Lwi6fr*57dz)Y6P(AnUEV(>lM*4KH$%xf6unJ6{%P^!+stVn zU!kU%KiM9Lac3!^p_78v=*;0qc(~o4sl&(-sA)&*+cs+)r6dOn^dSvb>ldt%8yxi@Ucfv0d2A0 z*1ET;xoCEyl6zO91lUe(!b^!b>w8V$trE}bz-LPWoNqQ@E-`BdNBtPJY{V8cY4aDK z8U(*Oa{F$g6I5xW2rm3xgbtNnbm%+J$H4+qS6-+HZ5{2I&+apxCjJl$D`cB1v?Uze zJW8Ejf6juZy5-v8?2-J0NIk~kkXO!@eFlTl(K@O@c6UYu5BWTFZA$vkrDS_+M6KI4 z6RU>4Ib?IlYO>68;=V>#L z@$PcqsS;dV0gFlv-bpK*s#UYw;m-nRn!g;^M{Uz}KG+yWtZuy;zDggu6sA7tLzT$b z)6*PcpVs(QZ+?6IE;%yD?CoM(QWF8;*DB(RJ3BNzJye&bZz%Fmvc~HCW*dC^T0wjd zC6RK_Y3h&TAi^&12Z>2K+kWjcd;M6^EGt$K$T?x>*H~9z5&!Fix6gp>vTo=PRWw$0 zv32AX&Pu8@GKh3bAQl5v!3$pwe=l__THe2VeSRo&jv)`6nad+-bN>~;5q9ANinaZa z(gFy-5`<6vWJ8lP*X&^D>EuJ!?#&}GRIBkq=p9?@g_b|77;bPXex(ztmB<{A|4X+2 zs>G#{^@FR9J9$0*pnZOD70WT~06T|=i5Ag~^3RxI9^Hel+d2qOr+@{Cnvo9WqvuMh zdheKMu{H?)t8O$EK}akF4P<&%k>%4y0<)H3Vvu9yz}Il z!kk_y3Ek4N0zL6K3RGisft6a*6%&(u#^$Ty)#JLIRyZZ~p;r71RB*tDB_uX?&A8 z%A{GJ#qXd;eMO8GQV)vpZ=ARt%J_0}2INZrBNFl!n^gxkG=GdNhS|Ws=Pv&=>Xb~3 z5INILdT>YMoZ=(D+`?4TEqR5uXMU;~4s#P;y9D3-ef3fqc+1`JH{nESJ2sVSPIdXa zv_{ppd7~Yg;DJo%vSN+tLwDd(9R%@DeYAZ!U-;O=9MFo7&kQQ;;{2-;&DYru-9Jsq zADHqpoLaxR{|l5511ZclZ9ZI7yQ2p)i5n~)HG~>@#4=?jC?M(5q}-?==u_oO;fN+% zP^2>JgVL=aQU3=voF^R~+pM}4+|C+gX%2V(k&nNQCSoBB6jcd2i;l;+*tO8pPsJ(R zCDYy!$Nzpc@%V=Tj_=G`_~mzfxKx$vV}%z3{TTSQP)|IClR63zVD(W199S6_D}6-= z?Z&%twBY0U7imu8-b|5ktai?^$9hf?o2;cMNpn?RtTWM`JO*FD>@&D(Izp)g z2{;D{(Lu8%Jt*inuON4W4LC*Wt+@HP|8RwPx@E|%)$)xcIkkWDmWhYQ2N?Jxv3_MT zbVS5rt;uVKIcUBwM$!)k2)4iu3{v!mOzyD?9?VFNYGrV&BBVYx(PcD2NHx#|U7gVFLJl zvbjaxuA*P^WdZSR`?#g9qtu;mQiK{I6V$5x>)9Lpfr+T_i!Pu^7*&+xCtk7n7QZBh zg-O!$wsyG}eR1NqHmXa08dbWU_r)meaO+opkuB1Hku48z=nwy6wo3*oMUmqVwFqhu z$_`KZ5tl;JW#_vLrqS1K2*nSoQE-29&Pm6t)h~3s8kViqmKf7Vb0PGP0iy^hem?+7 zp)SuYW2WQX362DOOnL19mgVaie^RLn*KL+~eW1xG`-r4A>RhaG6*~NOBy0W}-hxwc z4fp`x{p-y7$KsgnF`xtU<*&^M;>X#6n@1T=0p5;MsY0FQ91Aw^jkihNM3Np2#M*3W zcDGQw6}5B0yKUFakB|D7Xuu1@#$)0agI=i{u&(E}%pnX@Rgy!cF?ZUb<$z^K4a34%od%KI($dR#kk`s}v;LfaTvksXF?Vsg*JEPOzC7$Ek zZzi}nJu_!J;2~xLa zAms6%LVW!)01t~H@ELXlv0&Rb7}L|2HN=3Mh*6cCW#6+p^PODz3DU@^3+(U(_{G@| z4hu1n6M(s?yrUQLgC3?5=L*#M53bEkjF5J|ecE{Z*i^>#z3du2&O-;3@0i0|a;Y7T zo;bl5Vejd_s1h%atK6JLugo8}1*V|4ZHAr7FP@?7FTp^gB+gJ$c6V;LPc?wJMEU-h zuNRRx{U5LJch?TIVATcByUx5blbp6FNP2sBr0|q8@%6(BK6xk!?2AI5{4bYWh9r2T3GY+n8J&8;* z3<>L=L`ugkrO=*hi{J3_S~#`hxen=ZQ0uid=Qt_+oS`r2gd3H5>&A#BsY39A=WAyc z8a*GX457feDFO9#m&YE?Fu#%^_P^+nxkzjgAi4RLoQ z3~|(T{1e@Y_U0mu&m5B4%7;9b41&Ne?s?Dn%N83!s2<*25_}JHIfvtdg_q`XLfpFX z5cro;#UU{pL$&)ZC|eTg*6yrZ!s?$~SJ1G+qQ{6DPz zME*eq)ODO@GJb)*de$I~o{jYn?%WCJCRVqGinh6}bjcA<065^;`I zh@EnlTbg9h%q&u+oHRf4tX};YfXXi>Mr&VOs_=N80-{;q0m+XBT-`B91R>T;;7kWj z6s9aT2y8)7Ts>Il67IR=#pQWf7R_eKPrB9xRqmiuuXZwdM7QOg9K+Zz6KfjX{FKR=G23WLx~SYC1)OLaHCeOu6`Fc zBet|4)I>Mx!5 0: + # Send each record to OpenAI for schema extraction + openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/tesrt/completions?api-version=2023-09-15-preview' + openai_headers = { + 'Content-Type': 'application/json', + 'api-key': 'ebe64320148849aead404cc3aec9cc49' + } + + for record in json_data: + openai_payload = { + "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {record}", + "max_tokens": 800, + "temperature": 0.2, + "frequency_penalty": 0, + "presence_penalty": 0, + "top_p": 1, + "stop": None + } + + try: + openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) + openai_response.raise_for_status() + + # Extract only the text part from the OpenAI response + response_text = openai_response.json()['choices'][0]['text'] + print(response_text) + except requests.exceptions.HTTPError as errh: + print(f"HTTP Error: {errh}") + except requests.exceptions.RequestException as err: + print(f"Request Exception: {err}") + else: + print("No records found in the JSON data.") +else: + print(f"API call to fetch data failed with status code {response.status_code}") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1bc570e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +openai==0.28.0 +fuzzywuzzy==0.18.0 \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..7a311fc --- /dev/null +++ b/test.py @@ -0,0 +1,79 @@ +import requests +from pymongo import MongoClient +import json + +def fetch_sample_record(collection): + data = list(collection.find()) + if len(data) > 0: + return data[0] + return None + +# Connect to MongoDB for darvin_user.new_user collection +mongo_connection_string_darvin = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" +mongo_client_darvin = MongoClient(mongo_connection_string_darvin) +mongo_db_darvin = mongo_client_darvin['darvin_user'] +mongo_collection_darvin = mongo_db_darvin['new_user'] + +# Connect to MongoDB for bugfix-avi.user collection +# Connect to MongoDB for bugfix-avi.user collection +mongo_connection_string_bugfix = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" +mongo_client_bugfix = MongoClient(mongo_connection_string_bugfix) +mongo_db_bugfix = mongo_client_bugfix['bugfix-avi'] +mongo_collection_bugfix = mongo_db_bugfix['user'] + +# Fetch JSON data from darvin_user.new_user collection +sample_record_darvin = fetch_sample_record(mongo_collection_darvin) + +if sample_record_darvin: + # Send the sample record to OpenAI for schema extraction + openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' + openai_headers = { + 'Content-Type': 'application/json', + 'api-key': 'ebe64320148849aead404cc3aec9cc49' + } + + openai_payload = { + "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record_darvin}", + "max_tokens": 800, + "temperature": 0.2, + "frequency_penalty": 0, + "presence_penalty": 0, + "top_p": 1, + "stop": None + } + + try: + openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) + openai_response.raise_for_status() + + # Extract only the text part from the OpenAI response + openai_output = openai_response.json()['choices'][0]['text'] + + # Extract labels from the OpenAI output + openai_labels = [label.strip() for label in openai_output.split('\n') if label] + + # Fetch JSON data from bugfix-avi.user collection + sample_record_bugfix = fetch_sample_record(mongo_collection_bugfix) + + if sample_record_bugfix: + # Convert the JSON string to a dictionary + sample_record_bugfix = json.loads(sample_record_bugfix) + + # Extract labels from the sample record in bugfix-avi.user + bugfix_labels = [entry['label'] for entry in sample_record_bugfix] + + # Compare labels + similar_labels = set(openai_labels).intersection(set(bugfix_labels)) + + # Print similar labels + print("Similar Labels:") + print(similar_labels) + else: + print("No records found in bugfix-avi.user collection.") + except requests.exceptions.HTTPError as errh: + print(f"HTTP Error: {errh}") + except requests.exceptions.RequestException as err: + print(f"Request Exception: {err}") + +else: + print("No records found in the MongoDB collection.") diff --git a/test1.py b/test1.py new file mode 100644 index 0000000..be2f9b3 --- /dev/null +++ b/test1.py @@ -0,0 +1,92 @@ +from fastapi import FastAPI, Form, HTTPException +import openai +import json + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + # Extracting the mapped elements from the string response + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + for key, value in json_data.items(): + mapped_elements.append(f"{{\"l1 element\": \"{key}\", \"l2 element\": \"{value}\"}}") + + return mapped_elements + except (KeyError, ValueError): + return None + + +def chat_with_bot(l1, l2, syntax): + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + # Format the mapped elements from the response + formatted_elements = format_mapped_elements(response) + + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append(formatted_elements) # Automatically store the response + return formatted_elements + +# FastAPI endpoint for chatbot +@app.post("/chat") +def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + bot_reply = chat_with_bot(l1, l2, syntax) + return {"bot_reply": bot_reply} + +# FastAPI endpoint to get stored responses +@app.get("/get_responses") +def get_responses(): + if not stored_responses: + raise HTTPException(status_code=404, detail="No stored responses") + return {"stored_responses": stored_responses} + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/tt.txt b/tt.txt new file mode 100644 index 0000000..a58c580 --- /dev/null +++ b/tt.txt @@ -0,0 +1,62 @@ +Completionnnnnnnnn: { + "id": "chatcmpl-8LpWjW1XIuZoZAe6hvMLje1V19cdJ", + "object": "chat.completion", + "created": 1700213649, + "model": "gpt-35-turbo", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + }, + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 9, + "total_tokens": 34 + } +} \ No newline at end of file From 5313bcb3d9f323997e53bfbc9861e6bbe92c70c7 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 20 Dec 2023 16:56:39 +0530 Subject: [PATCH 02/91] Code syncf --- .gitignore | 3 + __init__.py | 4 + app.py | 8 ++ bot.py | 110 ++++++++++++++++ botchat.py | 130 +++++++++++++++++++ bott.py | 89 +++++++++++++ chat.py | 69 ++++++++++ chatbot.py | 118 +++++++++++++++++ config.yaml | 2 + config/loader.py | 23 ++++ database/connection.py | 23 ++++ fetch_labels.py | 284 +++++++++++++++++++++++++++-------------- func.py | 76 +++++++++++ his5.py | 103 --------------- index.html | 85 ++++++++++++ json_schema.py | 45 ------- mapping.py | 47 ------- regenrate.py | 105 --------------- requirements.txt | 25 ++++ s.html | 188 +++++++++++++++++++++++++++ script.js | 30 +++++ seed.txt | 1 + sp.html | 190 +++++++++++++++++++++++++++ ss.py | 42 ++++++ style.css | 35 +++++ templates/index.html | 227 ++++++++++++++++++++++++++++++++ templates/input.html | 68 ++++++++++ templates/output.html | 85 ++++++++++++ ui.py | 88 +++++++++++++ 29 files changed, 1909 insertions(+), 394 deletions(-) create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 app.py create mode 100644 bot.py create mode 100644 botchat.py create mode 100644 bott.py create mode 100644 chat.py create mode 100644 chatbot.py create mode 100644 config.yaml create mode 100644 config/loader.py create mode 100644 database/connection.py create mode 100644 func.py delete mode 100644 his5.py create mode 100644 index.html delete mode 100644 json_schema.py delete mode 100644 mapping.py delete mode 100644 regenrate.py create mode 100644 requirements.txt create mode 100644 s.html create mode 100644 script.js create mode 100644 seed.txt create mode 100644 sp.html create mode 100644 ss.py create mode 100644 style.css create mode 100644 templates/index.html create mode 100644 templates/input.html create mode 100644 templates/output.html create mode 100644 ui.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e53e071 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.vscode/ +__pycache__/ +myenv/ diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..a01d9ce --- /dev/null +++ b/__init__.py @@ -0,0 +1,4 @@ +import uvicorn + +if __name__ == "__main__": + uvicorn.run("app:app", host="127.0.0.1", port=8000, reload=True) \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..ee0176d --- /dev/null +++ b/app.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI, Form, HTTPException, Request +from fetch_labels import router as ProcessForm +from chat import router as ChatResponse + +app = FastAPI() + +app.include_router(ProcessForm, tags=["Labels"], prefix="/Openai/FetchLabel") +app.include_router(ChatResponse, tags=["ChatOpenai"], prefix="/Openai/chat") diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..e9c28a9 --- /dev/null +++ b/bot.py @@ -0,0 +1,110 @@ +import os +from fastapi import FastAPI, Form, HTTPException +import openai +import json + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +# Load or initialize the seed from a file +seed_file = "seed.txt" +if os.path.exists(seed_file): + with open(seed_file, "r") as file: + seed = int(file.read()) +else: + seed = 42 # Set a default seed if the file doesn't exist + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + # Extracting the mapped elements from the string process + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + for key, value in json_data.items(): + mapped_elements.append({"l1 element": key, "l2 element": value}) + return mapped_elements + except (KeyError, ValueError): + return None + +def chat_with_bot(l1, l2, syntax): + while True: + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.8, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None, + seed=seed # Use the seed value + ) + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + # Format the mapped elements from the response + formatted_elements = format_mapped_elements(response) + + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append(formatted_elements) # Automatically store the response + + return formatted_elements + + +# FastAPI endpoint for comparison and filtering +@app.post("/compare") +def compare_elements(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + # Fetch the complete element names from the original chat response + chat_response = chat_with_bot(l1, l2, syntax) + + # Extract complete element names for l1 and l2 + matching_elements = [ + {"l1": element["l2 element"]["l1"], "l2": element["l2 element"]["l2"]} + for element in chat_response + ] + + return {"matching_elements": matching_elements} + + + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) + +# Save the seed for the next run +with open(seed_file, "w") as file: + file.write(str(seed)) \ No newline at end of file diff --git a/botchat.py b/botchat.py new file mode 100644 index 0000000..536e0df --- /dev/null +++ b/botchat.py @@ -0,0 +1,130 @@ +import os +from fastapi import FastAPI, Form, HTTPException +import openai +import json +from fastapi.responses import HTMLResponse + + + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +# Load or initialize the seed from a file +seed_file = "seed.txt" +if os.path.exists(seed_file): + with open(seed_file, "r") as file: + seed = int(file.read()) +else: + seed = 42 # Set a default seed if the file doesn't exist + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + # Extracting the mapped elements from the string process + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + # Adjust the formatting to match the desired structure + for key, value in json_data.items(): + mapped_elements.append({"l1": key, "l2": value}) + + return mapped_elements + except (KeyError, ValueError): + return None + +def chat_with_bot(l1, l2, syntax): + while True: + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.6, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None, + seed=seed # Use the seed value + ) + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + # Format the mapped elements from the response + formatted_elements = format_mapped_elements(response) + + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append(formatted_elements) # Automatically store the response + + return formatted_elements + + +@app.post("/compare", response_model=dict) +def compare_elements(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + # Fetch the complete element names from the original chat response + chat_response = chat_with_bot(l1, l2, syntax) + + + temp_arr = [] + for element in chat_response: + print("mew") + print(element['l2']) + for element_val in element['l2']: + temp_dict = {} + temp_dict['element_name_l1'] = element_val['l1_element'] + temp_dict['element_name_l2'] = element_val['l2_element'] + temp_arr.append(temp_dict) + + + # Wrap the response within a dictionary key "matching_elements" + response = {"similar_elements": temp_arr} + print(response) # Print the response to terminal + + return response + + +#Route to serve the index.html file +@app.get("/", response_class=HTMLResponse) +async def serve_index(): + with open("s.html", "r") as file: + return file.read() + + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) + +# Save the seed for the next run +with open(seed_file, "w") as file: + file.write(str(seed)) \ No newline at end of file diff --git a/bott.py b/bott.py new file mode 100644 index 0000000..2fa606c --- /dev/null +++ b/bott.py @@ -0,0 +1,89 @@ +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse, HTMLResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn + +app = FastAPI() + + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + similarity = fuzz.ratio( + str(element_l1).lower(), str(element_l2).lower() + ) # Convert to lowercase for case-insensitive comparison + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + # print("Matching Elements in l1:", matching_elements_l1) + # print("Matching Elements in l2:", matching_elements_l2) + # print("Non-Matching Elements in l1:", non_matching_elements_l1) + # print("Non-Matching Elements in l2:", non_matching_elements_l2) + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + + +@app.post("/compare", response_model=dict) +def compare_lists( + l1: Union[str, List[str]] = Form(...), + l2: Union[str, List[str]] = Form(...), + threshold: int = Form(70, gt=0, le=100), +): + if isinstance(l1, str): + l1_list = convert_string_to_list(l1) + else: + l1_list = l1 + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + print("result: ",result) + + return JSONResponse(content=result) + + +# Route to serve the index.html file +@app.get("/", response_class=HTMLResponse) +async def serve_index(): + with open("templates/index.html", "r") as file: + return file.read() + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/chat.py b/chat.py new file mode 100644 index 0000000..2ec4df5 --- /dev/null +++ b/chat.py @@ -0,0 +1,69 @@ +import os +from fastapi import FastAPI, Form, HTTPException, Request +import openai +import json +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates +from fastapi import APIRouter +from fastapi import FastAPI, Request, Form +from fastapi.templating import Jinja2Templates +from fastapi.responses import HTMLResponse +from pydantic import BaseModel +import uvicorn +from func import extract_labels_from_record, get_response_text, format_mapped_elements, chat_with_bot +from motor.motor_asyncio import AsyncIOMotorClient + + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +#app = FastAPI() +router = APIRouter() +templates = Jinja2Templates(directory="templates") + + + +@router.post("/chat") +async def chat_endpoint(request: Request, syntax: str = Form(...), db_name2: str = Form(...), + collection_name2: str = Form(...), labels1: str = Form(...)): + l1 = labels1.split('|') if labels1 else [] + seed_value = 10 + + mongo_connection_string = ( + "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/" + "?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" + ) + mongo_client = AsyncIOMotorClient(mongo_connection_string) + mongo_db2 = mongo_client[db_name2] + mongo_collection2 = mongo_db2[collection_name2] + mongo_data2 = await mongo_collection2.find().to_list(length=None) + + + if len(mongo_data2) > 0: + sample_record2 = mongo_data2[0] + labels2 = extract_labels_from_record(sample_record2) + else: + labels2 = [] + + l2 = labels2 + + bot_reply = chat_with_bot(l1, l2, syntax,seed=seed_value) + + # Calculate matched and unmatched elements for L1 and L2 + l1_matched = [label for label in l1 if label in bot_reply] + l2_matched = [label for label in l2 if label in bot_reply] + + l1_unmatched = list(set(l1) - set(l1_matched)) + l2_unmatched = list(set(l2) - set(l2_matched)) + + # Now you have l1_matched, l2_matched, l1_unmatched, and l2_unmatched available for further use + return { + "bot_reply": bot_reply, + "l1_matched": l1_matched, + "l2_matched": l2_matched, + "l1_unmatched": l1_unmatched, + "l2_unmatched": l2_unmatched + } + #return {"bot_reply":bot_reply} \ No newline at end of file diff --git a/chatbot.py b/chatbot.py new file mode 100644 index 0000000..a49df0c --- /dev/null +++ b/chatbot.py @@ -0,0 +1,118 @@ +import os +from fastapi import FastAPI, Form, HTTPException +import openai +import json +from fastapi.responses import HTMLResponse + +openai.api_type = "azure" +openai.api_base = "https://cymetriopen.openai.azure.com/" +openai.api_version = "2023-07-01-preview" +openai.api_key = "ebe64320148849aead404cc3aec9cc49" + +app = FastAPI() + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + #Extracting the mapped elelments from the string process + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + for key,value in json_data.items(): + mapped_elements.append({"l1 element": key, "l2 element": value}) + return mapped_elements + except (KeyError,ValueError): + return None + + +def chat_with_bot(l1, l2, syntax): + while True: + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messagess`` + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.5, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None + ) + + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + try: + response_dict = json.loads(response) + similar_elements = response_dict.get("similar_elements", []) + except (KeyError, ValueError): + similar_elements = [] + + + # Update stored responses with extracted elements + formatted_elements = format_mapped_elements(response) + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append({"content": formatted_elements, "similar_elements": similar_elements}) + + return formatted_elements + + +#Route to serve the index.html file +@app.get("/", response_class=HTMLResponse) +async def serve_index(): + with open("sp.html", "r") as file: + return file.read() + + +# FastAPI endpoint for chatbot +# @app.post("/chat") +# def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): +# bot_reply = chat_with_bot(l1, l2, syntax) +# return {"bot_reply": bot_reply} + +@app.post("/chat") +def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): + bot_reply = chat_with_bot(l1, l2, syntax) + + # Update response dictionary with default L2 values + response_dict = {"bot_reply": bot_reply} + stored_response = stored_responses[-1] + for i, l1_element in enumerate(bot_reply): + for similar_element in stored_response["similar_elements"]: + if l1_element["l1 element"] == similar_element["l1_element"]: + response_dict["bot_reply"][i]["l2 element"] = similar_element["l2_element"] + break + print("response_dict: ",response_dict) + return response_dict + +# Run the FastAPI server +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..668c761 --- /dev/null +++ b/config.yaml @@ -0,0 +1,2 @@ +MONGODB_CONNECTION_STRING: mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority +CYMMETRI_DB_PREFIX: cymmetri- \ No newline at end of file diff --git a/config/loader.py b/config/loader.py new file mode 100644 index 0000000..33d83fe --- /dev/null +++ b/config/loader.py @@ -0,0 +1,23 @@ +import yaml +import platform + +class Configuration: + def __init__(self, filepath): + self.data = {} + with open(filepath, "r") as yamlfile: + data_loaded = yaml.safe_load(yamlfile) + + self.data = data_loaded + + def getConfiguration(self): + return self.data + +if platform.system() == 'Windows': + c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-rolemining/config.yaml") +else: + c = Configuration("config.yaml") + +ConfObject = c.getConfiguration() + +def getConfigObject(): + return ConfObject \ No newline at end of file diff --git a/database/connection.py b/database/connection.py new file mode 100644 index 0000000..abce394 --- /dev/null +++ b/database/connection.py @@ -0,0 +1,23 @@ +import os +from pymongo import MongoClient +from config.loader import Configuration +import platform + +def load_configuration(): + if platform.system() == 'Windows': + c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-genrativeAI/config.yaml") + else: + c = Configuration("config.yaml") + return c.getConfiguration() + +data = load_configuration() + +MONGO_DETAILS = data["MONGODB_CONNECTION_STRING"] +BASE_TENANT_STRING = data["CYMMETRI_DB_PREFIX"] + "%s" +mongo_client = MongoClient(MONGO_DETAILS) + +def get_collection(tenant_name: str, collection_name: str): + mongo_tenant_details = BASE_TENANT_STRING % tenant_name + database = mongo_client[mongo_tenant_details] + generic_collection = database[collection_name] + return generic_collection diff --git a/fetch_labels.py b/fetch_labels.py index 0c8b457..7261f77 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -1,94 +1,190 @@ -import requests -from pymongo import MongoClient - -def extract_labels_from_record(record): - - labels = [] - if isinstance(record, dict): - for key, value in record.items(): - label_entry = { - "jsonPath": key, - "label": key.replace("_", " ").title(), - "dataType": type(value).__name__, - "value": value - } - labels.append(label_entry["label"]) - return labels - -# Connect to MongoDB for darvin_user.new_user collection -mongo_connection_string_darvin = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" -mongo_client_darvin = MongoClient(mongo_connection_string_darvin) -mongo_db_darvin = mongo_client_darvin['darvin_user'] -mongo_collection_darvin = mongo_db_darvin['new_user'] - -# Connect to MongoDB for bugfix-avi.user collection -mongo_connection_string_bugfix = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" -mongo_client_bugfix = MongoClient(mongo_connection_string_bugfix) -mongo_db_bugfix = mongo_client_bugfix['bugfix-avi'] -mongo_collection_bugfix = mongo_db_bugfix['user'] - -# Fetch JSON data from darvin_user.new_user collection -mongo_data_darvin = list(mongo_collection_darvin.find()) - -if len(mongo_data_darvin) > 0: - sample_record_darvin = mongo_data_darvin[0] - - # Extract labels from the sample record in darvin_user.new_user - labels_darvin = extract_labels_from_record(sample_record_darvin) - - # Print labels from darvin_user.new_user - print("Labels from darvin_user.new_user:") - print(labels_darvin) - - # Fetch JSON data from bugfix-avi.user collection - mongo_data_bugfix = list(mongo_collection_bugfix.find()) - - if len(mongo_data_bugfix) > 0: - sample_record_bugfix = mongo_data_bugfix[0] - - # Extract labels from the sample record in bugfix-avi.user - labels_bugfix = extract_labels_from_record(sample_record_bugfix) - - # Print labels from bugfix-avi.user - print("Labels from bugfix-avi.user:") - print(labels_bugfix) - - # Send labels to OpenAI for finding similar labels - openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' - openai_headers = { - 'Content-Type': 'application/json', - 'api-key': 'ebe64320148849aead404cc3aec9cc49' - } - - openai_payload = { - "prompt": f"Find all labels in: {labels_darvin} and {labels_bugfix} and store in list name as l1 and l2", - "max_tokens": 800, - "temperature": 0.2, - "frequency_penalty": 0, - "presence_penalty": 0, - "top_p": 1, - "stop": None - } - - try: - openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() - - # Extract only the text part from the OpenAI response - json_response_from_openai = openai_response.json() - similar_labels = openai_response.json()['choices'][0]['text'] - print("Similar Labels:") - print(similar_labels) - - - #print("-=------------") - #print(json_response_from_openai) - except requests.exceptions.HTTPError as errh: - print(f"HTTP Error: {errh}") - except requests.exceptions.RequestException as err: - print(f"Request Exception: {err}") - - else: - print("No records found in bugfix-avi.user collection.") -else: - print("No records found in darvin_user.new_user collection.") +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException +from fastapi.templating import Jinja2Templates +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse, HTMLResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union + +app = FastAPI() +templates = Jinja2Templates(directory="templates") + +#logging.basicConfig(level=logging.DEBUG) + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + similarity = fuzz.ratio( + str(element_l1).lower(), str(element_l2).lower() + ) # Convert to lowercase for case-insensitive comparison + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + # print("Matching Elements in l1:", matching_elements_l1) + # print("Matching Elements in l2:", matching_elements_l2) + # print("Non-Matching Elements in l1:", non_matching_elements_l1) + # print("Non-Matching Elements in l2:", non_matching_elements_l2) + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + + +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: + final_response = [] + processed_labels = set() + + # Create a dictionary for easy lookup of response_data based on labels + response_lookup = {data['label']: data for data in response_data} + + for element in similar_elements: + # Find matching element in response_data based on label + matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) + + if matched_data: + final_response.append({ + 'jsonPath': matched_data['jsonPath'], + 'l2_matched': element['element_name_l2'], + 'datatype': matched_data['dataType'], + 'value': matched_data['value'] + }) + processed_labels.add(element['element_name_l1']) # Track processed labels + + # Handle unmatched elements from l1 + for data in response_data: + if data['label'] not in processed_labels: + final_response.append({ + 'jsonPath': data['jsonPath'], + 'l2_matched': '', # No match from l2 + 'datatype': data['dataType'], + 'value': data['value'] # Use value from response_data + }) + + return final_response + + +@app.post('/process_data') +async def process_data(request: Request, data_url: str = Form(...)): + try: + #logging.debug(f"Processing data from URL: {data_url}") + + # Fetch JSON data from the specified URL using httpx for asynchronous requests + async with httpx.AsyncClient() as client: + response = await client.get(data_url) + + if response.status_code == 200: + # Assuming the response contains JSON data, you can parse it + json_data = response.json() + + if isinstance(json_data, list) and len(json_data) > 0: + sample_record = json_data[0] + + # Send the sample record to OpenAI for schema extraction + openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' + openai_headers = { + 'Content-Type': 'application/json', + 'api-key': 'ebe64320148849aead404cc3aec9cc49' + } + + openai_payload = { + "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record}", + "max_tokens": 800, + "temperature": 0.2, + "frequency_penalty": 0, + "presence_penalty": 0, + "top_p": 1, + "stop": None + } + + try: + async with httpx.AsyncClient() as openai_client: + openai_response = await openai_client.post(openai_url, headers=openai_headers, json=openai_payload) + openai_response.raise_for_status() + + # Extract only the text part from the OpenAI response + response_text = openai_response.json()['choices'][0]['text'] + + # Parse the response_text into a list of dictionaries + response_data = json.loads(response_text) + + #logging.debug(f"Received response from OpenAI: {response_data}") + + + l1 = [item['label'] for item in response_data] + + if isinstance(l1, str): + l1_list = convert_string_to_list(l1) + else: + l1_list = l1 + + l2 = [' Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + threshold = 65 + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + final_response = generate_final_response(result['similar_elements'], response_data) + + return JSONResponse(content=final_response) + + except httpx.HTTPError as errh: + #logging.error(f"HTTP Error: {errh}") + raise HTTPException(status_code=500, detail=f"HTTP Error: {errh}") + except httpx.RequestError as err: + #logging.error(f"Request Exception: {err}") + raise HTTPException(status_code=500, detail=f"Request Exception: {err}") + else: + return "No records found in the JSON data." + else: + raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + except Exception as e: + #logging.error(f"An unexpected error occurred: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + return JSONResponse(content=result) + + + + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/func.py b/func.py new file mode 100644 index 0000000..a6a653d --- /dev/null +++ b/func.py @@ -0,0 +1,76 @@ +import json +import openai + +def extract_labels_from_record(record): + labels = [] + if isinstance(record, dict): + for key in record.keys(): + if isinstance(key, str): + label_entry = key.replace("_", " ").title().strip() + labels.append(label_entry) + return labels + +def get_response_text(response): + try: + # Attempt to get the text content from the response + text_content = response['choices'][0]['message']['content'] + return text_content + except (KeyError, IndexError): + return None + +def format_mapped_elements(response): + mapped_elements = [] + try: + #Extracting the mapped elelments from the string process + start_index = response.find("{") + end_index = response.rfind("}") + json_content = response[start_index:end_index + 1] + json_data = json.loads(json_content) + + for key,value in json_data.items(): + mapped_elements.append({"l1 element": key, "l2 element": value}) + return mapped_elements + except (KeyError,ValueError): + return None + + +# Maintain conversation history +conversation_history = [] +stored_responses = [] + +def chat_with_bot(l1, l2, syntax,seed=None): + while True: + # Concatenate l1, l2, and syntax into a single input string + input_text = f"{l1}\n{l2}\n{syntax}" + + # Initialize the conversation with system messages + message_text = [ + {"role": "system", "content": "You are an AI assistant that helps people find information."}, + ] + + # Append user inputs to the conversation + message_text.append({"role": "user", "content": input_text}) + + # Call OpenAI to get a response + completion = openai.ChatCompletion.create( + engine="tesrt", + messages=message_text, + temperature=0.7, + max_tokens=800, + top_p=0.95, + frequency_penalty=0, + presence_penalty=0, + stop=None, + seed=seed + ) + + # Get the assistant's response + response = get_response_text(completion) + print("Response received from OpenAI:", response) + + # Format the mapped elements from the response + formatted_elements = format_mapped_elements(response) + + conversation_history.append({"role": "assistant", "content": formatted_elements}) + stored_responses.append(formatted_elements) # Automatically store the response + return formatted_elements \ No newline at end of file diff --git a/his5.py b/his5.py deleted file mode 100644 index b81d511..0000000 --- a/his5.py +++ /dev/null @@ -1,103 +0,0 @@ -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -# Flag to control bot response -continue_conversation = True - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Simulate an error in err.py -error_found = False -try: - exec(open("java_err.java").read()) -except Exception as e: - error_message = str(e) - print("error_msg: ", error_message) - error_found = True - # Use the error message as input for the chatbot - openai_solution_prompt = f"Solve the error: {error_message}" - openai_solution_response = chat_with_bot(openai_solution_prompt) - print("first_res_bot: ", openai_solution_response) - - user_input = input("Do you want to continue the conversation? (yes/no): ") - - if user_input.lower() == "yes": - while continue_conversation: - print("You: ") - user_input_lines = [] - - while True: - line = input() - if line.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - # Combine multiline input into a single string - user_input = '\n'.join(user_input_lines) - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - - response = chat_with_bot(user_input) - print("continue_resp: ",response) - - # Use fuzzy logic to compare the current input with the previous responses - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break - - elif user_input.lower() == "no": - print("Chatbot: Goodbye!") - continue_conversation == False - - -if not error_found: - print("No error found") - diff --git a/index.html b/index.html new file mode 100644 index 0000000..98ba9b6 --- /dev/null +++ b/index.html @@ -0,0 +1,85 @@ + + + + Matched Fields UI + + + + +

+
+
    + +
+
+
+
    + +
+
+ +
+ + + + diff --git a/json_schema.py b/json_schema.py deleted file mode 100644 index 74f417f..0000000 --- a/json_schema.py +++ /dev/null @@ -1,45 +0,0 @@ -import requests - -# Fetch JSON data from the specified URL -data_url = 'https://jsonplaceholder.typicode.com/users' -response = requests.get(data_url) - -if response.status_code == 200: - # Assuming the response contains JSON data, you can parse it - json_data = response.json() - - if isinstance(json_data, list) and len(json_data) > 0: - sample_record = json_data[0] - - # Send the sample record to OpenAI for schema extraction - openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' - openai_headers = { - 'Content-Type': 'application/json', - 'api-key': 'ebe64320148849aead404cc3aec9cc49' - } - - openai_payload = { - "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record}", - "max_tokens": 800, - "temperature": 0.2, - "frequency_penalty": 0, - "presence_penalty": 0, - "top_p": 1, - "stop": None - } - - try: - openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() - - # Extract only the text part from the OpenAI response - response_text = openai_response.json()['choices'][0]['text'] - print(response_text) - except requests.exceptions.HTTPError as errh: - print(f"HTTP Error: {errh}") - except requests.exceptions.RequestException as err: - print(f"Request Exception: {err}") - else: - print("No records found in the JSON data.") -else: - print(f"API call to fetch data failed with status code {response.status_code}") diff --git a/mapping.py b/mapping.py deleted file mode 100644 index cba5ff1..0000000 --- a/mapping.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests -from pymongo import MongoClient - -# Connect to MongoDB (replace the connection string with your actual MongoDB connection string) -mongo_connection_string = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" -mongo_client = MongoClient(mongo_connection_string) - -# Specify the database and collection from which you want to fetch data -mongo_db = mongo_client['darvin_user'] -mongo_collection = mongo_db['new_user'] - -# Fetch JSON data from MongoDB -mongo_data = list(mongo_collection.find()) - -if len(mongo_data) > 0: - sample_record = mongo_data[0] - - # Send the sample record to OpenAI for schema extraction - openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' - openai_headers = { - 'Content-Type': 'application/json', - 'api-key': 'ebe64320148849aead404cc3aec9cc49' - } - - openai_payload = { - "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record}", - "max_tokens": 800, - "temperature": 0.2, - "frequency_penalty": 0, - "presence_penalty": 0, - "top_p": 1, - "stop": None - } - - try: - openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() - - # Extract only the text part from the OpenAI response - response_text = openai_response.json()['choices'][0]['text'] - print(response_text) - except requests.exceptions.HTTPError as errh: - print(f"HTTP Error: {errh}") - except requests.exceptions.RequestException as err: - print(f"Request Exception: {err}") -else: - print("No records found in the MongoDB collection.") diff --git a/regenrate.py b/regenrate.py deleted file mode 100644 index e86fb47..0000000 --- a/regenrate.py +++ /dev/null @@ -1,105 +0,0 @@ -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - print("Bot:", response) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - -def regenerate_response(previous_question): - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=[ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - {"role": "user", "content": previous_question} - ], - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - return response - -print("Chatbot: Hello! How can I assist you today?") - -while True: - print("You :") - user_input_lines = [] - - while True: - line = input() - if line.lower() == 'exit': - break - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - user_input = '\n'.join(user_input_lines) - - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - bot_reply = chat_with_bot(user_input) - - if user_input.lower() == 'regenerate': - previous_question = None - for message in reversed(conversation_history): - if message["role"] == "assistant": - previous_question = message['content'] - break - - if previous_question: - regenerated_response = regenerate_response(previous_question) - print("Chatbot: Here's a different response linked to your previous question:") - print("Bot:", regenerated_response) - conversation_history.append({"role": "assistant", "content": regenerated_response}) - else: - print("Chatbot: Sorry, there's no previous question to generate a response.") - continue - - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..83ab3eb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,25 @@ +fastapi==0.100.1 +pandas==2.0.3 +motor==3.2.0 +pydantic==2.1.1 +uvicorn==0.23.2 +annotated-types==0.5.0 +anyio==3.7.1 +click==8.1.6 +colorama==0.4.6 +dnspython==2.4.1 +h11==0.14.0 +idna==3.4 +pip==23.1.2 +pydantic_core==2.4.0 +pymongo==4.4.1 +python-dateutil==2.8.2 +pytz==2023.3 +setuptools==65.5.0 +six==1.16.0 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.7.1 +tzdata==2023.3 +python-multipart +PyYAML==6.0 \ No newline at end of file diff --git a/s.html b/s.html new file mode 100644 index 0000000..a1ff5a9 --- /dev/null +++ b/s.html @@ -0,0 +1,188 @@ + + + + + Chat with Bot + + + +

Chat with Bot

+
+ + + + + + + + + + +
+ +
+ + + + diff --git a/script.js b/script.js new file mode 100644 index 0000000..475768b --- /dev/null +++ b/script.js @@ -0,0 +1,30 @@ +const list1 = ["FirstName", "Display Name", "Email Address", "Id", "address"]; +const list2 = ["fullname", "firstname", "company mail", "employeeId", "office address"]; + +const ul1 = document.getElementById("list1"); +const ul2 = document.getElementById("list2"); +const lines = document.getElementById("lines"); + +list1.forEach((item, index) => { + const li = document.createElement("li"); + li.textContent = item; + ul1.appendChild(li); + + const matchingIndex = list2.indexOf(item); + if (matchingIndex !== -1) { + const line = document.createElementNS("http://www.w3.org/2000/svg", "line"); + line.setAttribute("x1", "0"); + line.setAttribute("y1", `${li.offsetTop + li.offsetHeight / 2}`); + line.setAttribute("x2", "100%"); + line.setAttribute("y2", `${ul2.children[matchingIndex].offsetTop + ul2.children[matchingIndex].offsetHeight / 2}`); + line.setAttribute("stroke", "#000"); + line.setAttribute("stroke-width", "2"); + lines.appendChild(line); + } +}); + +list2.forEach((item, index) => { + const li = document.createElement("li"); + li.textContent = item; + ul2.appendChild(li); +}); diff --git a/seed.txt b/seed.txt new file mode 100644 index 0000000..f70d7bb --- /dev/null +++ b/seed.txt @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/sp.html b/sp.html new file mode 100644 index 0000000..9bbc807 --- /dev/null +++ b/sp.html @@ -0,0 +1,190 @@ + + + + + Chat with Bot + + + +

Chat with Bot

+
+ + + + + + + + + + +
+ +
+
+ + + + \ No newline at end of file diff --git a/ss.py b/ss.py new file mode 100644 index 0000000..f2bb89e --- /dev/null +++ b/ss.py @@ -0,0 +1,42 @@ +similar_elements = [{'element_name_l1': ' Id', 'element_name_l2': ' Id'}, {'element_name_l1': 'First Name', 'element_name_l2': 'Firstname'}, {'element_name_l1': 'Last Name', 'element_name_l2': 'Lastname'}, {'element_name_l1': 'Hod Email Id', 'element_name_l2': 'Email'}, {'element_name_l1': 'Job Role', 'element_name_l2': 'Mobile'}, {'element_name_l1': 'Full Name', 'element_name_l2': 'Lastname'}, {'element_name_l1': 'Marital Status', 'element_name_l2': 'Status'}, {'element_name_l1': 'Office Country', 'element_name_l2': 'Country'}, {'element_name_l1': 'Current Country', 'element_name_l2': 'Country'}, {'element_name_l1': 'Current State', 'element_name_l2': 'Created'}] + + +def parse_elements(elements_list): + l1_matched = [] + l2_matched = [] + + for element in elements_list: + l1_matched.append(element['element_name_l1']) + l2_matched.append(element['element_name_l2']) + + return l1_matched, l2_matched + +l1_matched, l2_matched = parse_elements(similar_elements) +print("L1 Matched:", l1_matched) +print() +print("L2 Matched:", l2_matched) + +def find_unmatched_elements(original_list, matched_list): + unmatched = [element for element in original_list if element not in matched_list] + return unmatched + +# Example usage: +l1_original = ['Id','First Name','Last Name','Band','Employee Id','Business Unit','Ctc','Cost Center Id','Date Of Exit','Date Of Joining','Date Of Activation','Departments Hierarchy','Direct Manager Employee Id','Employee Type','Grade','Group Company','Hod Email Id','Hod','Hod Employee Id','Hrbp Employee Id', 'Job Role', 'Job Role Code', 'Job Role Name', 'Job Level Code', 'L2 Manager Employee Id', 'Separation Agreed Last Date', 'Separation Status', 'Total Ctc', 'Candidate Id', 'Center Type', 'Company Email Id', 'Date Of Birth', 'Date Of Resignation', 'City Type', 'Dependents', 'Education Details', 'Emergency Contact Number', 'Emergency Contact Person', 'Emergency Contact Relation', 'Full Name', 'Gender', 'Location Type', 'Marital Status', 'Middle Name', 'Office Address', 'Office Area', 'Office City', 'Office Country', 'Office Location', 'Office State', 'Primary Mobile Number', 'User Unique Id', 'Work Area Code', 'Current Address', 'Current Country', 'Current State', 'Personal Email Id', 'Personal Mobile No', 'Office Mobile No', 'Bank Name', 'Bank Branch', 'Bank Account', 'Bank Ifsc', 'Bank Pan', 'Aadhaar Number', 'Job Level', 'Function', 'Leadership Group', 'Nationality', 'Function Code', 'Direct Manager Email', 'Direct Manager Name', 'Permanent Pin Code', 'Latest Modified Any Attribute', 'Regular/ Temporary', 'Blood Group', 'Departments Hierarchy With Codes', 'Notice Period Assigned', 'Salutation', 'Permanent Address', 'Permanent City', 'Permanent Country', 'Permanent State', 'Date Of Death', 'Anniversary Date', 'Group Company Code', 'Functional Head', 'Employment Type', 'Contract End Date', 'Contract Start Date', 'Attendance Shift', 'Role', 'Dependent Role', 'Cost Center', 'Office Location Cost Center', 'Is Rehire', 'Rehire By', 'Rehire On', 'Rehire Reason', 'Employee Separation Reason', 'Admin Deactivation Type', 'Dm Pool', 'Home', 'Sub Employee Type', 'Hrbp Email Id', 'Source', 'Source Type', 'Fnf Status', 'Past Work', 'Passport Address', 'Passport Country', 'Passport Number', 'Passport Type', 'Federation Id', "Father'S Name", 'Job Role Alias', 'Channel', 'Fls/Nfls', 'Short Desg.', 'Zone', 'Region', 'Recruiter', 'Zrm', 'Hrbp', 'Zhr', 'Uan Number', 'Esic Number', 'Pf Number', 'Activation Timestamp', 'Designation Title', 'Latest Modified Timestamp'] + +l2_original = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + +l1_matched = ['Id', 'First Name', 'Last Name', 'Hod Email Id', 'Job Role', 'Full Name', 'Marital Status', 'Office Country', 'Current Country', 'Current State'] +l2_matched = [' Id', 'Firstname', 'Lastname', 'Email', 'Mobile', 'Lastname', 'Status', 'Country', 'Country', 'Created'] + + +l1_unmatched = find_unmatched_elements(l1_original, [elem.strip() for elem in l1_matched]) +l2_unmatched = find_unmatched_elements(l2_original, [elem.strip() for elem in l2_matched]) + +print() +print("Unmatched Elements L1:", l1_unmatched) +print() +print("Unmatched Elements L2:", l2_unmatched) + + + diff --git a/style.css b/style.css new file mode 100644 index 0000000..af10e32 --- /dev/null +++ b/style.css @@ -0,0 +1,35 @@ +body { + font-family: Arial, sans-serif; + } + + .container { + display: flex; + justify-content: space-between; + align-items: center; + height: 100vh; + padding: 20px; + } + + .list { + border: 1px solid #ccc; + border-radius: 5px; + padding: 10px; + flex: 1; + max-height: 80vh; + overflow-y: auto; + } + + .left { + margin-right: 10px; + } + + .right { + margin-left: 10px; + } + + svg { + position: absolute; + top: 0; + left: 0; + } + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..20039dc --- /dev/null +++ b/templates/index.html @@ -0,0 +1,227 @@ + + + + + Chat with Bot + + + +

Chat with Bot

+
+ + + + + + + + + + +
+ +
+
+ + + + \ No newline at end of file diff --git a/templates/input.html b/templates/input.html new file mode 100644 index 0000000..1d0e55f --- /dev/null +++ b/templates/input.html @@ -0,0 +1,68 @@ + + + + + + Data Processing + + + +

Data Processing

+
+ + + +
+ + \ No newline at end of file diff --git a/templates/output.html b/templates/output.html new file mode 100644 index 0000000..7e4f831 --- /dev/null +++ b/templates/output.html @@ -0,0 +1,85 @@ + + + + + + Data Processing Result + + + +

Data Processing Result

+ + + + + + + + + + + {% for record in response_data %} + + + + + + + {% endfor %} + +
jsonPathlabeldataTypevalue
{{ record.jsonPath }}{{ record.label }}{{ record.dataType }}{{ record.value }}
+ Go Back + + \ No newline at end of file diff --git a/ui.py b/ui.py new file mode 100644 index 0000000..e521706 --- /dev/null +++ b/ui.py @@ -0,0 +1,88 @@ +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse, HTMLResponse +from fuzzywuzzy import fuzz +from typing import List, Union + +app = FastAPI() + + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + similarity = fuzz.ratio( + str(element_l1).lower(), str(element_l2).lower() + ) # Convert to lowercase for case-insensitive comparison + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + # print("Matching Elements in l1:", matching_elements_l1) + # print("Matching Elements in l2:", matching_elements_l2) + # print("Non-Matching Elements in l1:", non_matching_elements_l1) + # print("Non-Matching Elements in l2:", non_matching_elements_l2) + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + + +@app.post("/compare", response_model=dict) +def compare_lists( + l1: Union[str, List[str]] = Form(...), + l2: Union[str, List[str]] = Form(...), + threshold: int = Form(70, gt=0, le=100), +): + if isinstance(l1, str): + l1_list = convert_string_to_list(l1) + else: + l1_list = l1 + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + return JSONResponse(content=result) + + +# Route to serve the index.html file +@app.get("/", response_class=HTMLResponse) +async def serve_index(): + with open("sp.html", "r") as file: + return file.read() + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file From ee104aceb2e36d5fe21282d3acac1272a21d0554 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 20 Dec 2023 19:10:46 +0530 Subject: [PATCH 03/91] changes made for returning response of users. --- fetch_labels.py | 112 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/fetch_labels.py b/fetch_labels.py index 7261f77..2fc27a6 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -104,9 +104,115 @@ async def process_data(request: Request, data_url: str = Form(...)): # Fetch JSON data from the specified URL using httpx for asynchronous requests async with httpx.AsyncClient() as client: - response = await client.get(data_url) + ''' + + + body +: +"{\n \"size\":10,\n \"page\":1,\n \"filter\":\"*\"\n}" +headers +: +Array(2) +0 +: +{key: 'Authorization', value: 'Bearer afsafasf', included: true} +1 +: +{key: 'Content-Type', value: 'application/json', included: true} +length +: +2 +[[Prototype]] +: +Array(0) +id +: +1 +label +: +"Node 1" +params +: +[] +parentProperties +: +[] +requestMethod +: +"GET" +response +: +"" +selectedNodeObj +: +[[Prototype]] +: +Object +selectedNodes +: +Array(0) +length +: +0 +[[Prototype]] +: +Array(0) +url +: +"" +[[Prototype]] +: +Object + ''' + + ### From lokesh's request -- you will get 4 things + # applicationId + # body --> which you have to convert from JSON string to dictionary + # headers --> convert from [] to dictionary + # url --> use as-is (will be string) + # requestMethod --> will be a string; use with match as given below else use if-else ladder + + + # check if headers is empty + ### Parse body --- {\n \"size\":10,\n \"page\":1,\n \"filter\":\"*\"\n} and take it as dictionary + #### Get url directly from requests + ### Get requestMethod from requestMethod + + ### you will get this as an array; convert each element of array + + body_dict = {} + header_dict = {} + url_from_request = "" + requestMethod_from_request = "" + + + client.headers = header_dict + + + + ### into dict; such that key in the array element is your dict key and value is corresponding value + match requestMethod_from_request: + case "GET": + # Call method for GET request + response = client.get(url=request.data.url) + case "POST": + # Call method for POST request + ## if body is empty, skip the data field + ## if body is not empty, use as below + response = client.post(url=request.data.url, data=body_dict) + case "PUT": + # Handle other request methods + response = client.put(url=request.data.url, data=body_dict) + + case "PATCH": + response = client.patch(url=request.data.url, data=body_dict) + + case "DELETE": + response = client.delete(url=request.data.url) + + #response = await client.get(data_url) - if response.status_code == 200: + if response.status_code >= 200 or response.status_code <= 204: # Assuming the response contains JSON data, you can parse it json_data = response.json() @@ -179,8 +285,6 @@ async def process_data(request: Request, data_url: str = Form(...)): except Exception as e: #logging.error(f"An unexpected error occurred: {e}") raise HTTPException(status_code=500, detail=str(e)) - - return JSONResponse(content=result) From 2a2c5f45f0efbbb24a4c1273044bf37899fc4d58 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Thu, 21 Dec 2023 15:42:17 +0530 Subject: [PATCH 04/91] Removed pip from requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 029ab62..1ebba1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ colorama==0.4.6 dnspython==2.4.1 h11==0.14.0 idna==3.4 -pip==23.1.2 pydantic_core==2.4.0 pymongo==4.4.1 python-dateutil==2.8.2 From fbf639f2c330dca92ca01b81743f94da1ef187a7 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 26 Dec 2023 11:06:41 +0530 Subject: [PATCH 05/91] removed unwanted files --- Dockerfile | 22 ++++ HelloWorld.java | 5 - LICENSE | 21 ---- README.md | 1 - __init__.py | 4 - api_json.py | 69 ------------- api_json1.py | 71 ------------- app.py | 8 -- bot.py | 110 -------------------- botchat.py | 130 ------------------------ bott.py | 89 ----------------- chat-simple.py | 66 ------------ chat.py | 69 ------------- chatbot.py | 118 ---------------------- err.py | 34 ------- fetch_list.py | 84 ---------------- fn_json_api.py | 94 ----------------- func.py | 76 -------------- his1.py | 86 ---------------- his2.py | 105 ------------------- his3.py | 80 --------------- his4.py | 83 --------------- his5.py | 106 -------------------- his6.py | 103 ------------------- history.py | 73 -------------- index.html | 85 ---------------- input.txt | 1 - java_err.java | 37 ------- json_schema.png | Bin 50232 -> 0 bytes list_op.py | 29 ------ main.py | 44 -------- main2.py | 44 -------- requirements.txt | 26 ----- s.html | 188 ---------------------------------- script.js | 30 ------ sp.html | 190 ----------------------------------- ss.py | 42 -------- style.css | 35 ------- templates/index.html | 227 ------------------------------------------ templates/input.html | 68 ------------- templates/output.html | 85 ---------------- test.py | 79 --------------- test1.py | 92 ----------------- tt.txt | 62 ------------ 44 files changed, 22 insertions(+), 3049 deletions(-) create mode 100644 Dockerfile delete mode 100644 HelloWorld.java delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 __init__.py delete mode 100644 api_json.py delete mode 100644 api_json1.py delete mode 100644 app.py delete mode 100644 bot.py delete mode 100644 botchat.py delete mode 100644 bott.py delete mode 100644 chat-simple.py delete mode 100644 chat.py delete mode 100644 chatbot.py delete mode 100644 err.py delete mode 100644 fetch_list.py delete mode 100644 fn_json_api.py delete mode 100644 func.py delete mode 100644 his1.py delete mode 100644 his2.py delete mode 100644 his3.py delete mode 100644 his4.py delete mode 100644 his5.py delete mode 100644 his6.py delete mode 100644 history.py delete mode 100644 index.html delete mode 100644 input.txt delete mode 100644 java_err.java delete mode 100644 json_schema.png delete mode 100644 list_op.py delete mode 100644 main.py delete mode 100644 main2.py delete mode 100644 requirements.txt delete mode 100644 s.html delete mode 100644 script.js delete mode 100644 sp.html delete mode 100644 ss.py delete mode 100644 style.css delete mode 100644 templates/index.html delete mode 100644 templates/input.html delete mode 100644 templates/output.html delete mode 100644 test.py delete mode 100644 test1.py delete mode 100644 tt.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..085223e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.10-slim + +# Set the working directory to /app +WORKDIR /app + +# Copy the contents of the local directory into the container at /app +COPY . /app + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Install Flask +RUN pip install flask + +# Make port 5000 available to the world outside this container +EXPOSE 5000 + +# Define environment variable +ENV NAME World + +# Run your FastAPI application +CMD ["uvicorn", "fetch_labels:app", "--host", "0.0.0.0", "--port", "5000"] diff --git a/HelloWorld.java b/HelloWorld.java deleted file mode 100644 index 70fd330..0000000 --- a/HelloWorld.java +++ /dev/null @@ -1,5 +0,0 @@ -public class HelloWorld { - public static void main(String[] args) { - System.out.println("Hello, World!"); - } -} diff --git a/LICENSE b/LICENSE deleted file mode 100644 index c4d19bf..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Kathreftis AI PVT LTD - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index 9850a5c..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# cymmetri-microservices-genrativeAI \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index a01d9ce..0000000 --- a/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import uvicorn - -if __name__ == "__main__": - uvicorn.run("app:app", host="127.0.0.1", port=8000, reload=True) \ No newline at end of file diff --git a/api_json.py b/api_json.py deleted file mode 100644 index 6a7b6be..0000000 --- a/api_json.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException - -import openai - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - # Initialize the conversation with system and user messages - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - {"role": "user", "content": user_input}, - ] + conversation_history - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Get the assistant's response - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - stored_responses.append(response) # Automatically store the response - return response - -# FastAPI endpoint for chatbot -@app.post("/chat") -def chat_endpoint(user_input: str = Form(...)): - bot_reply = chat_with_bot(user_input) - return {"bot_reply": bot_reply} - -# FastAPI endpoint to get stored responses -@app.get("/get_responses") -def get_responses(): - if not stored_responses: - raise HTTPException(status_code=404, detail="No stored responses") - return {"stored_responses": stored_responses} - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/api_json1.py b/api_json1.py deleted file mode 100644 index 8714e4d..0000000 --- a/api_json1.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException -import openai - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(l1, l2, syntax): - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Get the assistant's response - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - stored_responses.append(response) # Automatically store the response - return response - -# FastAPI endpoint for chatbot -@app.post("/chat") -def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - bot_reply = chat_with_bot(l1, l2, syntax) - return {"bot_reply": bot_reply} - -# FastAPI endpoint to get stored responses -@app.get("/get_responses") -def get_responses(): - if not stored_responses: - raise HTTPException(status_code=404, detail="No stored responses") - return {"stored_responses": stored_responses} - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/app.py b/app.py deleted file mode 100644 index ee0176d..0000000 --- a/app.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI, Form, HTTPException, Request -from fetch_labels import router as ProcessForm -from chat import router as ChatResponse - -app = FastAPI() - -app.include_router(ProcessForm, tags=["Labels"], prefix="/Openai/FetchLabel") -app.include_router(ChatResponse, tags=["ChatOpenai"], prefix="/Openai/chat") diff --git a/bot.py b/bot.py deleted file mode 100644 index e9c28a9..0000000 --- a/bot.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException -import openai -import json - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -# Load or initialize the seed from a file -seed_file = "seed.txt" -if os.path.exists(seed_file): - with open(seed_file, "r") as file: - seed = int(file.read()) -else: - seed = 42 # Set a default seed if the file doesn't exist - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - # Extracting the mapped elements from the string process - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - for key, value in json_data.items(): - mapped_elements.append({"l1 element": key, "l2 element": value}) - return mapped_elements - except (KeyError, ValueError): - return None - -def chat_with_bot(l1, l2, syntax): - while True: - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.8, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None, - seed=seed # Use the seed value - ) - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - # Format the mapped elements from the response - formatted_elements = format_mapped_elements(response) - - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append(formatted_elements) # Automatically store the response - - return formatted_elements - - -# FastAPI endpoint for comparison and filtering -@app.post("/compare") -def compare_elements(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - # Fetch the complete element names from the original chat response - chat_response = chat_with_bot(l1, l2, syntax) - - # Extract complete element names for l1 and l2 - matching_elements = [ - {"l1": element["l2 element"]["l1"], "l2": element["l2 element"]["l2"]} - for element in chat_response - ] - - return {"matching_elements": matching_elements} - - - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) - -# Save the seed for the next run -with open(seed_file, "w") as file: - file.write(str(seed)) \ No newline at end of file diff --git a/botchat.py b/botchat.py deleted file mode 100644 index 536e0df..0000000 --- a/botchat.py +++ /dev/null @@ -1,130 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException -import openai -import json -from fastapi.responses import HTMLResponse - - - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -# Load or initialize the seed from a file -seed_file = "seed.txt" -if os.path.exists(seed_file): - with open(seed_file, "r") as file: - seed = int(file.read()) -else: - seed = 42 # Set a default seed if the file doesn't exist - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - # Extracting the mapped elements from the string process - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - # Adjust the formatting to match the desired structure - for key, value in json_data.items(): - mapped_elements.append({"l1": key, "l2": value}) - - return mapped_elements - except (KeyError, ValueError): - return None - -def chat_with_bot(l1, l2, syntax): - while True: - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.6, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None, - seed=seed # Use the seed value - ) - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - # Format the mapped elements from the response - formatted_elements = format_mapped_elements(response) - - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append(formatted_elements) # Automatically store the response - - return formatted_elements - - -@app.post("/compare", response_model=dict) -def compare_elements(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - # Fetch the complete element names from the original chat response - chat_response = chat_with_bot(l1, l2, syntax) - - - temp_arr = [] - for element in chat_response: - print("mew") - print(element['l2']) - for element_val in element['l2']: - temp_dict = {} - temp_dict['element_name_l1'] = element_val['l1_element'] - temp_dict['element_name_l2'] = element_val['l2_element'] - temp_arr.append(temp_dict) - - - # Wrap the response within a dictionary key "matching_elements" - response = {"similar_elements": temp_arr} - print(response) # Print the response to terminal - - return response - - -#Route to serve the index.html file -@app.get("/", response_class=HTMLResponse) -async def serve_index(): - with open("s.html", "r") as file: - return file.read() - - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) - -# Save the seed for the next run -with open(seed_file, "w") as file: - file.write(str(seed)) \ No newline at end of file diff --git a/bott.py b/bott.py deleted file mode 100644 index 2fa606c..0000000 --- a/bott.py +++ /dev/null @@ -1,89 +0,0 @@ -from fastapi import FastAPI, Form -from fastapi.responses import JSONResponse, HTMLResponse -from fuzzywuzzy import fuzz -from typing import List, Union -import uvicorn - -app = FastAPI() - - -def convert_string_to_list(input_str: str) -> List[str]: - # Remove leading and trailing whitespaces, and split by ',' - return [element.strip() for element in input_str.strip('[]').split(',')] - - -def compare_lists_with_fuzzy(l1, l2, threshold=50): - matching_elements_l1 = [] - matching_elements_l2 = [] - non_matching_elements_l1 = [] - non_matching_elements_l2 = [] - - for element_l1 in l1: - max_similarity = 0 - matching_element_l2 = '' - - for element_l2 in l2: - similarity = fuzz.ratio( - str(element_l1).lower(), str(element_l2).lower() - ) # Convert to lowercase for case-insensitive comparison - if similarity > max_similarity and similarity >= threshold: - max_similarity = similarity - matching_element_l2 = element_l2 - - if matching_element_l2: - matching_elements_l1.append(element_l1.strip("'")) - matching_elements_l2.append(matching_element_l2.strip("'")) - else: - non_matching_elements_l1.append(element_l1.strip("'")) - - non_matching_elements_l2 = [ - element_l2.strip("'") - for element_l2 in l2 - if element_l2.strip("'") not in matching_elements_l2 - ] - - # print("Matching Elements in l1:", matching_elements_l1) - # print("Matching Elements in l2:", matching_elements_l2) - # print("Non-Matching Elements in l1:", non_matching_elements_l1) - # print("Non-Matching Elements in l2:", non_matching_elements_l2) - - similar_elements = [] - for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) - - result = {"similar_elements": similar_elements} - return result - - -@app.post("/compare", response_model=dict) -def compare_lists( - l1: Union[str, List[str]] = Form(...), - l2: Union[str, List[str]] = Form(...), - threshold: int = Form(70, gt=0, le=100), -): - if isinstance(l1, str): - l1_list = convert_string_to_list(l1) - else: - l1_list = l1 - - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 - - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - - print("result: ",result) - - return JSONResponse(content=result) - - -# Route to serve the index.html file -@app.get("/", response_class=HTMLResponse) -async def serve_index(): - with open("templates/index.html", "r") as file: - return file.read() - - -if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/chat-simple.py b/chat-simple.py deleted file mode 100644 index 41f10be..0000000 --- a/chat-simple.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import openai - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -#Maintain conversation history -conversation_history = [] - -continue_conversation = True - - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - # Initialize the conversation with system and user messages - conversation_history.append({"role": "user","content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - {"role": "user", "content": user_input}, - ] + conversation_history - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Print the assistant's response - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - print("Bot:", response) - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Chat loop -while True: - # Get user input - user_input = input("You: ") - - # Check if the user wants to exit the conversation - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - # Generate response - bot_reply = chat_with_bot(user_input) - # compare this two fields based on fuzzy logic and give me matching fields from this two list \ No newline at end of file diff --git a/chat.py b/chat.py deleted file mode 100644 index 6273679..0000000 --- a/chat.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException, Request -import openai -import json -from fastapi.responses import HTMLResponse -from fastapi.templating import Jinja2Templates -from fastapi import APIRouter -from fastapi import FastAPI, Request, Form -from fastapi.templating import Jinja2Templates -from fastapi.responses import HTMLResponse -from pydantic import BaseModel -import uvicorn -from func import extract_labels_from_record, get_response_text, format_mapped_elements, chat_with_bot -from motor.motor_asyncio import AsyncIOMotorClient - - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -#app = FastAPI() -router = APIRouter() -templates = Jinja2Templates(directory="templates") - - - -@router.post("/chat") -async def chat_endpoint(request: Request, syntax: str = Form(...), db_name2: str = Form(...), - collection_name2: str = Form(...), labels1: str = Form(...)): - l1 = labels1.split('|') if labels1 else [] - seed_value = 10 - - mongo_connection_string = ( - "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/" - "?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" - ) - mongo_client = AsyncIOMotorClient(mongo_connection_string) - mongo_db2 = mongo_client[db_name2] - mongo_collection2 = mongo_db2[collection_name2] - mongo_data2 = await mongo_collection2.find().to_list(length=None) - - - if len(mongo_data2) > 0: - sample_record2 = mongo_data2[0] - labels2 = extract_labels_from_record(sample_record2) - else: - labels2 = [] - - l2 = labels2 - - bot_reply = chat_with_bot(l1, l2, syntax,seed=seed_value) - - # Calculate matched and unmatched elements for L1 and L2 - l1_matched = [label for label in l1 if label in bot_reply] - l2_matched = [label for label in l2 if label in bot_reply] - - l1_unmatched = list(set(l1) - set(l1_matched)) - l2_unmatched = list(set(l2) - set(l2_matched)) - - # Now you have l1_matched, l2_matched, l1_unmatched, and l2_unmatched available for further use - return { - "bot_reply": bot_reply, - "l1_matched": l1_matched, - "l2_matched": l2_matched, - "l1_unmatched": l1_unmatched, - "l2_unmatched": l2_unmatched - } - #return {"bot_reply":bot_reply} diff --git a/chatbot.py b/chatbot.py deleted file mode 100644 index a49df0c..0000000 --- a/chatbot.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException -import openai -import json -from fastapi.responses import HTMLResponse - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - #Extracting the mapped elelments from the string process - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - for key,value in json_data.items(): - mapped_elements.append({"l1 element": key, "l2 element": value}) - return mapped_elements - except (KeyError,ValueError): - return None - - -def chat_with_bot(l1, l2, syntax): - while True: - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messagess`` - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.5, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - try: - response_dict = json.loads(response) - similar_elements = response_dict.get("similar_elements", []) - except (KeyError, ValueError): - similar_elements = [] - - - # Update stored responses with extracted elements - formatted_elements = format_mapped_elements(response) - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append({"content": formatted_elements, "similar_elements": similar_elements}) - - return formatted_elements - - -#Route to serve the index.html file -@app.get("/", response_class=HTMLResponse) -async def serve_index(): - with open("sp.html", "r") as file: - return file.read() - - -# FastAPI endpoint for chatbot -# @app.post("/chat") -# def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): -# bot_reply = chat_with_bot(l1, l2, syntax) -# return {"bot_reply": bot_reply} - -@app.post("/chat") -def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - bot_reply = chat_with_bot(l1, l2, syntax) - - # Update response dictionary with default L2 values - response_dict = {"bot_reply": bot_reply} - stored_response = stored_responses[-1] - for i, l1_element in enumerate(bot_reply): - for similar_element in stored_response["similar_elements"]: - if l1_element["l1 element"] == similar_element["l1_element"]: - response_dict["bot_reply"][i]["l2 element"] = similar_element["l2_element"] - break - print("response_dict: ",response_dict) - return response_dict - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/err.py b/err.py deleted file mode 100644 index 88bb5d1..0000000 --- a/err.py +++ /dev/null @@ -1,34 +0,0 @@ -# # Sample Python code with IndexError -# numbers = [1, 2, 3] -# print(numbers[3]) # This will result in an IndexError - - -my_list = [1, 2, 3] -my_list.append(4) -print(my_list.upper()) - - -#indentetion error -# if True: -# print("Indented block") - -#Key error -# my_dict = {"key": "value"} -# print(my_dict["nonexistent_key"]) - - -#ValueError -# int("abc") - -#Name Error -# x = 10 -# print(y) - - -#ModuleNotFoundError -#import non_existent_module - - - - - diff --git a/fetch_list.py b/fetch_list.py deleted file mode 100644 index 3526159..0000000 --- a/fetch_list.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -continue_conversation = True - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def compare_lists_fuzzy(l1, l2): - matching_elements = [] - for item1 in l1: - for item2 in l2: - # Using fuzzy ratio to compare elements - similarity_ratio = fuzz.ratio(item1, item2) - # You can adjust the threshold based on your requirements - if similarity_ratio > 80: # Adjust the threshold as needed - matching_elements.append((item1, item2, similarity_ratio)) - return matching_elements - -def chat_with_bot(user_input): - # Initialize the conversation with system and user messages - conversation_history.append({"role": "user", "content": user_input}) - - if "l1" in user_input.lower() and "l2" in user_input.lower(): - # Extract lists from user input - l1 = input("Enter elements for list l1 (comma-separated): ").split(",") - l2 = input("Enter elements for list l2 (comma-separated): ").split() - - # Compare lists based on fuzzy logic - matching_elements = compare_lists_fuzzy(l1, l2) - response = f"The matching elements based on fuzzy logic are: {matching_elements}" - print("Bot:", response) - else: - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - {"role": "user", "content": user_input}, - ] + conversation_history - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Print the assistant's response - response = get_response_text(completion) - conversation_history.append({"role": "assistant", "content": response}) - print("Bot:", response) - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Chat loop -while True: - # Get user input - user_input = input("You: ") - - # Check if the user wants to exit the conversation - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - # Generate response - bot_reply = chat_with_bot(user_input) diff --git a/fn_json_api.py b/fn_json_api.py deleted file mode 100644 index 0cc6771..0000000 --- a/fn_json_api.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -from fastapi import FastAPI, Form, HTTPException -import openai -import json - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - #Extracting the mapped elelments from the string process - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - for key,value in json_data.items(): - mapped_elements.append(f"{{\"l1 element\": \"{key}\", \"l2 element\": \"{value}\"}}") - return mapped_elements - except (KeyError,ValueError): - return None - - -def chat_with_bot(l1, l2, syntax): - while True: - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - # Format the mapped elements from the response - formatted_elements = format_mapped_elements(response) - - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append(formatted_elements) # Automatically store the response - return formatted_elements - - -# FastAPI endpoint for chatbot -@app.post("/chat") -def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - bot_reply = chat_with_bot(l1, l2, syntax) - return {"bot_reply": bot_reply} - -# FastAPI endpoint to get stored responses -@app.get("/get_responses") -def get_responses(): - if not stored_responses: - raise HTTPException(status_code=404, detail="No stored responses") - return {"stored_responses": stored_responses} - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/func.py b/func.py deleted file mode 100644 index a6a653d..0000000 --- a/func.py +++ /dev/null @@ -1,76 +0,0 @@ -import json -import openai - -def extract_labels_from_record(record): - labels = [] - if isinstance(record, dict): - for key in record.keys(): - if isinstance(key, str): - label_entry = key.replace("_", " ").title().strip() - labels.append(label_entry) - return labels - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - #Extracting the mapped elelments from the string process - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - for key,value in json_data.items(): - mapped_elements.append({"l1 element": key, "l2 element": value}) - return mapped_elements - except (KeyError,ValueError): - return None - - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def chat_with_bot(l1, l2, syntax,seed=None): - while True: - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None, - seed=seed - ) - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - # Format the mapped elements from the response - formatted_elements = format_mapped_elements(response) - - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append(formatted_elements) # Automatically store the response - return formatted_elements \ No newline at end of file diff --git a/his1.py b/his1.py deleted file mode 100644 index 3cd4bf3..0000000 --- a/his1.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - # Append user message to conversation history - conversation_history.append({"role": "user", "content": user_input}) - - # Initialize the conversation with system and user messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Print the assistant's response - response = get_response_text(completion) - print("Bot:", response) - - # Append bot message to conversation history - conversation_history.append({"role": "assistant", "content": response}) - - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Chat loop -while True: - # Get user input - print("You: ") - user_input_lines = [] - - # Allow multiline input until two consecutive empty lines or 'exit' is entered - while True: - line = input() - if line.lower() == 'exit': - break - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - # Combine multiline input into a single string - user_input = '\n'.join(user_input_lines) - - # Check if the user wants to exit the conversation - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - # Generate response - bot_reply = chat_with_bot(user_input) - - # Use fuzzy logic to compare the current input with the previous responses - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break \ No newline at end of file diff --git a/his2.py b/his2.py deleted file mode 100644 index 0d8a27c..0000000 --- a/his2.py +++ /dev/null @@ -1,105 +0,0 @@ -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - print("Bot:", response) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - -def regenerate_response(previous_question): - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=[ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - {"role": "user", "content": previous_question} - ], - temperature=0.4, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - return response - -print("Chatbot: Hello! How can I assist you today?") - -while True: - print("You :") - user_input_lines = [] - - while True: - line = input() - if line.lower() == 'exit': - break - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - user_input = '\n'.join(user_input_lines) - - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - bot_reply = chat_with_bot(user_input) - - if user_input.lower() == 'regenerate': - previous_question = None - for message in reversed(conversation_history): - if message["role"] == "assistant": - previous_question = message['content'] - break - - if previous_question: - regenerated_response = regenerate_response(previous_question) - print("Chatbot: Here's a different response linked to your previous question:") - print("Bot:", regenerated_response) - conversation_history.append({"role": "assistant", "content": regenerated_response}) - else: - print("Chatbot: Sorry, there's no previous question to generate a response.") - continue - - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break \ No newline at end of file diff --git a/his3.py b/his3.py deleted file mode 100644 index ab1de46..0000000 --- a/his3.py +++ /dev/null @@ -1,80 +0,0 @@ -# his2.py - Chatbot code - -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - print("Bot:", response) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Simulate an error in err.py -error_found = False -try: - exec(open("err.py").read()) -except Exception as e: - error_message = str(e) - print("errorrrrr: ", error_message) - error_found = True - # Use the error message as input for the chatbot - print("Chatbot: It seems an error occurred. Let me assist you with that.") - bot_reply = chat_with_bot(error_message) - -if not error_found: - print("No error found") - -# Continue the conversation loop (optional) -while True: - print("You: ") - user_input = input() - - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - bot_reply = chat_with_bot(user_input) - - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break diff --git a/his4.py b/his4.py deleted file mode 100644 index 2354a9e..0000000 --- a/his4.py +++ /dev/null @@ -1,83 +0,0 @@ -# his2.py - Chatbot code - -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Simulate an error in err.py -error_found = False -try: - exec(open("java_err.java").read()) -except Exception as e: - error_message = str(e) - print("error_msg: ", error_message) - error_found = True - # Use the error message as input for the chatbot - #bot_reply = chat_with_bot(error_message) - - # If there is an error, fetch a solution from OpenAI - openai_solution_prompt = f"Solve the error: {error_message}" - openai_solution_response = chat_with_bot(openai_solution_prompt) - print("bot: ",openai_solution_response) - -if not error_found: - print("No error found") - -# Continue the conversation loop (optional) -while True: - print("You: ") - user_input = input() - - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - bot_reply = chat_with_bot(user_input) - - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break diff --git a/his5.py b/his5.py deleted file mode 100644 index 4bed3f1..0000000 --- a/his5.py +++ /dev/null @@ -1,106 +0,0 @@ -import openai -from fuzzywuzzy import fuzz -import traceback - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -# Flag to control bot response -continue_conversation = True - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input, code=None): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - if code: - message_text.append({"role": "user", "content": f"Here is the code:\n{code}"}) - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Simulate an error in err.py -error_found = False -try: - code = open("err.py").read() - exec(code) -except Exception as e: - error_message = str(e) - print("error_msg: ", error_message) - error_found = True - # Use the error message as input for the chatbot - openai_solution_prompt = f"Solve the error: {error_message}" - openai_solution_response = chat_with_bot(openai_solution_prompt, code) - print("Bot: ", openai_solution_response) - - user_input = input("Do you want to continue the conversation? (yes/no): ") - - if user_input.lower() == "yes": - while continue_conversation: - print("You: ") - user_input_lines = [] - - while True: - line = input() - if line.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - # Combine multiline input into a single string - user_input = '\n'.join(user_input_lines) - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - - response = chat_with_bot(user_input) - print("Bot: ",response) - - # Use fuzzy logic to compare the current input with the previous responses - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break - - elif user_input.lower() == "no": - print("Chatbot: Goodbye!") - continue_conversation == False - - -if not error_found: - print("No error found") diff --git a/his6.py b/his6.py deleted file mode 100644 index 2d3eacb..0000000 --- a/his6.py +++ /dev/null @@ -1,103 +0,0 @@ -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -# Flag to control bot response -continue_conversation = True - -def get_response_text(response): - try: - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - conversation_history.append({"role": "user", "content": user_input}) - - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - response = get_response_text(completion) - - conversation_history.append({"role": "assistant", "content": response}) - - return response - - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Simulate an error in err.py -error_found = False -try: - exec(open("err.py").read()) -except Exception as e: - error_message = str(e) - print("error_msg: ", error_message) - error_found = True - # Use the error message as input for the chatbot - openai_solution_prompt = f"Solve the error: {error_message}" - openai_solution_response = chat_with_bot(openai_solution_prompt) - print("Bot: ", openai_solution_response) - - user_input = input("Do you want to continue the conversation? (yes/no): ") - - if user_input.lower() == "yes": - while continue_conversation: - print("You: ") - user_input_lines = [] - - while True: - line = input() - if line.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - elif not line.strip() and user_input_lines and not user_input_lines[-1].strip(): - break - user_input_lines.append(line) - - # Combine multiline input into a single string - user_input = '\n'.join(user_input_lines) - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - exit() # Terminate the program - - response = chat_with_bot(user_input) - print("Bot: ",response) - - # Use fuzzy logic to compare the current input with the previous responses - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break - - elif user_input.lower() == "no": - print("Chatbot: Goodbye!") - continue_conversation == False - - -if not error_found: - print("No error found") - diff --git a/history.py b/history.py deleted file mode 100644 index fb0e2ad..0000000 --- a/history.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import openai -from fuzzywuzzy import fuzz - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -# Maintain conversation history -conversation_history = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def chat_with_bot(user_input): - # Append user message to conversation history - conversation_history.append({"role": "user", "content": user_input}) - - # Initialize the conversation with system and user messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] + conversation_history - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Print the assistant's response - response = get_response_text(completion) - print("Bot:", response) - - # Append bot message to conversation history - conversation_history.append({"role": "assistant", "content": response}) - - return response - -# Initial message -print("Chatbot: Hello! How can I assist you today?") - -# Chat loop -while True: - # Get user input - user_input = input("You: ") - - # Check if the user wants to exit the conversation - if user_input.lower() == 'exit': - print("Chatbot: Goodbye!") - break - - # Generate response - bot_reply = chat_with_bot(user_input) - - #use fuzzy logic to compare the current input with the previous responses - for message in reversed(conversation_history): - if message["role"] == "assistant": - similarity = fuzz.ratio(user_input.lower(), message["content"].lower()) - if similarity > 70: - print(f"Chatbot: It seems like you've asked a similar question before. Here's a related response: {message['content']}") - break diff --git a/index.html b/index.html deleted file mode 100644 index 98ba9b6..0000000 --- a/index.html +++ /dev/null @@ -1,85 +0,0 @@ - - - - Matched Fields UI - - - - -
-
-
    - -
-
-
-
    - -
-
- -
- - - - diff --git a/input.txt b/input.txt deleted file mode 100644 index 255244b..0000000 --- a/input.txt +++ /dev/null @@ -1 +0,0 @@ -l1 = [' Id', 'First Name', 'Last Name', 'Band', 'Employee Id', 'Business Unit', 'Ctc', 'Cost Center Id', 'Date Of Exit', 'Date Of Joining', 'Date Of Activation', 'Departments Hierarchy', 'Direct Manager Employee Id', 'Employee Type', 'Grade', 'Group Company', 'Hod Email Id', 'Hod', 'Hod Employee Id', 'Hrbp Employee Id', 'Job Role', 'Job Role Code', 'Job Role Name', 'Job Level Code', 'L2 Manager Employee Id', 'Separation Agreed Last Date', 'Separation Status', 'Total Ctc', 'Candidate Id', 'Center Type', 'Company Email Id', 'Date Of Birth', 'Date Of Resignation', 'City Type', 'Dependents', 'Education Details', 'Emergency Contact Number', 'Emergency Contact Person', 'Emergency Contact Relation', 'Full Name', 'Gender', 'Location Type', 'Marital Status', 'Middle Name', 'Office Address', 'Office Area', 'Office City', 'Office Country', 'Office Location', 'Office State', 'Primary Mobile Number', 'User Unique Id', 'Work Area Code', 'Current Address', 'Current Country', 'Current State', 'Personal Email Id', 'Personal Mobile No', 'Office Mobile No', 'Bank Name', 'Bank Branch', 'Bank Account', 'Bank Ifsc', 'Bank Pan', 'Aadhaar Number', 'Job Level', 'Function', 'Leadership Group', 'Nationality', 'Function Code', 'Direct Manager Email', 'Direct Manager Name', 'Permanent Pin Code', 'Latest Modified Any Attribute', 'Regular/ Temporary', 'Blood Group', 'Departments Hierarchy With Codes', 'Notice Period Assigned', 'Salutation', 'Permanent Address', 'Permanent City', 'Permanent Country', 'Permanent State', 'Date Of Death', 'Anniversary Date', 'Group Company Code', 'Functional Head', 'Employment Type', 'Contract End Date', 'Contract Start Date', 'Attendance Shift', 'Role', 'Dependent Role', 'Cost Center', 'Office Location Cost Center', 'Is Rehire', 'Rehire By', 'Rehire On', 'Rehire Reason', 'Employee Separation Reason', 'Admin Deactivation Type', 'Dm Pool', 'Home', 'Sub Employee Type', 'Hrbp Email Id', 'Source', 'Source Type', 'Fnf Status', 'Past Work', 'Passport Address', 'Passport Country', 'Passport Number', 'Passport Type', 'Federation Id', "Father'S Name", 'Job Role Alias', 'Channel', 'Fls/Nfls', 'Short Desg.', 'Zone', 'Region', 'Recruiter', 'Zrm', 'Hrbp', 'Zhr', 'Uan Number', 'Esic Number', 'Pf Number', 'Activation Timestamp', 'Designation Title', 'Latest Modified Timestamp'] l2 = [' Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] find the similar elements using fuzzy wuzzy logic and specify which element from l1 is matching with l2 and i dont want code just give me result. \ No newline at end of file diff --git a/java_err.java b/java_err.java deleted file mode 100644 index 08d43bc..0000000 --- a/java_err.java +++ /dev/null @@ -1,37 +0,0 @@ -// import java.util.Stack; - -// public class StackExample { -// public static void main(String[] args) { -// Stack stack = new Stack<>(); -// // Attempting to pop an element from an empty stack -// String element = stack.pop(); // This will throw EmptyStackException -// } -// } - -// UnsupportedOperationException: -// import java.util.Stack; - -// public class StackExample { -// public static void main(String[] args) { -// Stack stack = new Stack<>(); -// // Attempting to use an unsupported method -// stack.add("element"); // This will throw UnsupportedOperationException -// } -// } - - -//NullPointerException -// Exception in thread "main" java.lang.NullPointerException -// at com.example.MyClass.myMethod(MyClass.java:10) -// at com.example.MyClass.main(MyClass.java:5) - - - -Exception in thread "main" java.io.FileNotFoundException: file.txt (No such file or directory) - at java.base/java.io.FileInputStream.open0(Native Method) - at java.base/java.io.FileInputStream.open(FileInputStream.java:219) - at java.base/java.io.FileInputStream.(FileInputStream.java:157) - at com.example.FileExample.main(FileExample.java:8) - - - diff --git a/json_schema.png b/json_schema.png deleted file mode 100644 index 967a25c87065fb89a77bf718c85b0bad2ac44860..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50232 zcmeFZWl&t*y8Ro1CU_taoZt{7xVuBp;2H?--Z&u;+({s~yF0-N!GpUt?$CHcBYhk4 zzI*R;{%6s`zKG} zF;I|TTcE}!FJb?|Ijc&GJt-e1-i3WYFc(!2ee$F#8uiu)5%&3+gN(NGlP7r84?l3S zs#M3YjVYFDS}yPHEFBa~U5reOT#R7bo;>lQ_-JG4plokx=i*Gkr|U)GPQk~-!Sj}x zhli7ui;=ekjIoNrbSy?%F|C6tIIM|puIe2;h+V{VW{f}?; zeSTbI2tRq!MI$RAs^)HRaEJa*P4gc4zTFQhlfbIZSpQiCSvC-zN?f)5i_{nWnrW_Z zOohr$1g<6ZdAiF2TTX4xm?zI9EY`$b-`JM=$h{uJjjOYoFy^*RvI^AaHhK~9;b#CL zNf{dl17VU6a*)=eJ5A$y#2fNcbaoF*mLV`8WKO-cBF6iws4s16p zB;SXfe*S-Srx++LToE)Q-LG`wq|mfZX7*JoNt zvb`|t2?flIHm-lPVHbi8ll@l0HWIpX`TT3S0o7-rmAykz&20sj-4WRO&C`Ri5LX$jz^=O$O;dzj88|P#ao&$dY)_W#kGH=%hb!n+X=ELEk z)$jegODSdt^|t0TrdexvF|aF3_2D;(WpDA`2bEFFr0^n7caKnzitpG%o@xv0&tyET z0F{Jnq%fvsQjv_ND4ByHNc3h?w?`UwB)F_9GtP$HO${ndoM^YYci&)%m{?K1Jqb0r=tv?17qBY}uM04GHAYXY`Zxie>`PMUe6bV9-F6hp6&_Vy9)+N@(D;Smfw-!q4rey{U7rY)7~`zv=HdeTPeAeCjBqv@HVQuWZ<(+GD!j zDb&R%k#U-97a8cct6N+z9hmzZ)Xpd-5Hxmi?Yp4|$+!>vnHXH(RY5`W=aQHxTE`Cc z;nSvRv*j~PwVl;-em=y<87GJ8_7NyIr?7=Z+`AXE?QY_Y_58$d&HqZ+{|NMUTc$*j zL*nfIbr>hB_BtS?w^rJYXLEm)e33zv5C~`ACC|1a?D$%|EZQ-{_Bt`5K@!)ixhF zfjVi7UDIQB8ENa!DUJX_3E#RC$F(+QI|h=DvcrSB$_1y=n@os1ffkUeAf zIkk1y0-6Q#NWY5BkP+D&riy$R&W^3efV5SYiw&y9gppc zWUy+=htJliO7V!2j|!uk6xDX`F5nE>%i<`ug6r315;nDO57&tJ991P@3+&m{ZL$JD z-`yLJ0UJ|N=mmJb;I;FPPVIuo^&D{6xPLzSAaN#yZo2SX!>sUT`PGidp|rV*>y4j` zQXKcUpp_!-^{z8RZfP|vsawig@A&25H;Dj}-BMf}UKNT!xSP3rZC=<*Q>DC=&` z1WUytzwJ_--TR|PSqn~Bi3klHxpI{{eUIULbU=ZKhRE{nkwIh_@1a_x|c#D{I3c&4F+aC1A6Lg6SB%*!yGe4!dPCEpqy< zqBEZxJI(yUynybKab=lS+n=@Y?h00~|A^x@C3DGGZzUkGpns#W z-^h|K;uO%OWf9R~udfo@YNPEC{2tKN6wMCtdni6@aY>5_qe zHir+56Ie*ECPVUPM;}hN4AF?M)vR_TXL$GTCB;cH-q2QXi|K1{wEg;d&^1h1}mKbi;u-QkyDrb@Savvb2siXOE#cX!s$ z0$QaLRW5SI=ooz+_X-Z_?S!|_oYZjd$S0=60)|)dMO-xs#im0YiR zy~uc95likjQV(twH7rIy0Y_-|SDo!RVRTL{_(O0epn*O0l1s#F?(gN@e>0l{)b~m;#MabjA=9v`q>$URBX;tvUC3UB^wz|N>bUL)M)L#C4(T{ zI!`{m$cg5JE`|yomyF(JEUVUwR?Ks~+*j?i$5j=*(DdUAaGIo|0=g;w_r72}pK%Ew38u!CmKDNQ*?DzE8+uoODXDPP zmaNqG75Ttv3Cm6p1Hm#gA(NJ;4J7?WMv`D~RQU>NzCOXd;9G3TF450hL8YwX{U#MXX=Gx>oHUtnd7{x{v-;!= zxjEDbu{|dP%?gQg6XTT(2`{2c%93h0g-cQ%QkX#lcY(} zgJsW3MiS0E{nfkn+qW!x^VgTxm(o~>rNt%d&`wJspQ;aSds9jTdGM(|-OUl^`z63A zi*|%V#U*lA(Y+8hD+T&r1^XE@APVb=uove=Z7io^ZE{i)l&r@}apTkkoxwb5vHpwH zOxKvRL9VEI`<9G?Jj(&j@$)=F`oKUc1sAoSOO?*sV>s#B%{2_BqrABN4LUQE0Yn z+6o6P#Vp~?zNi^HHA}$0N=qDjRmu|Jfk@{mE3CriC1kLCOgjQhF5EI?-e*hpA5UyO zeD~Z&(qf6aet(tDoC2r%-9fw>)o1_)GscGN#T&Y`=I?1o2tJp7wHV%?PcB)>3DU4V z8U}p;C@0R{{l6_n>_>E?-)#1Th8w#{G`S8m>r+VsVc&!Rj_g_BK@>*d$fRwR7ql&x-k_@Hg*P1WAL(?E>2Z#2CcS%s&(i7kiUc9iL*YZeQI&w7>t#3Jo#A^vP=rhm*i(qa+l8M=N_VDn&uAohD0;+n~f%# zjN*3!J#Y@I4-N>ZcjOyV#tJiNIy)3ntsz1dqls%wLN}df`eLc}P2XLr)%>b4z7>>r zQiF5!FvK11x6NBLnd@aSn{X_td-+;6t$WZwvgp`fzHr`GCUJHvTMbLRu6(RI9Oj}a zH%)A!KLB(lAb(d)ub-`B6Q_pKxt*v&wY;VBP;DkJXQebGaD{kl&yEAH7Dl8w&CU&;QpJ4|JB z1!Iyl3YR;!rYsP~o{c#Q`k?eup$qe=|GX2-+FDRc?yD!co#tn(xJ&Gd{)|6#EhSME z48HoNG156VN3_tD5k!-GW_!DEDZb!H1)}Rs6(q}amkZJ~^PtS$#fkR^a^_apVtTMN zu03n_7Jiw{3v9HNkXVO()9%;7^Q0NcSG^)WZ?c}0i65{!eQzbeQ(AD>?6_HR?G7+m z;3H#K{_sXbSNjW~b+5sZ?_<=xcWziBP+9!DXF|o&mwH#0Oc6iQXE^;0>b%sAdG-4w z-f1tcr_ZwNQfyXs`rl*}F-fx=zoI;aI3U{D61UoRsD0z6$Ua#i7c_dSYbjj+vSq}f z?G7$CG=JF;4%5sT+wqcFnZn)Wy zGzGcF0E^a%D27RQN8T7%7$fxlfb1JNg5{Kw1g?U6v|lyavj6(zGn)ndP~PU^n3e*m zcj|p5*5d6gzxkc9pHG{=8iXWOd~Un(MBaPzCi24VTTYQ~Id1Uz=L`24EoI)Ox}PWz zwhmCSc!EtOU;~~CWb^5+TqYysOBi>}RP{swiJjy9+KB~3x7zPAvX8P(DdH~s)CvvV zemB#v;r(F>?-qoApsId1zX}UmV`*?Ou59*g}|DKPd$=D$eKQiuxKKG zW7flkH+W_5B`pW-yz3s0i7d7TQvy4{VM`>GXaMl&GWnLht22MmVwAQf?3$lCFk!8*id6g z@{AC=V%WMM=@t`yLd4`h*`>vIuo!iZS}+Mmz2kM;e?er!-u?U7wU$4kc7(5-AUd9D z_}5sdzQlKzf6)#S_p%bWgl;SB7>3YRv>)j`jzVe2XqcUcyvv5->**}6%EMCDht^W) z`pRj?SEbu+{DD_LvcryM=ECRvqdCxKE(}+{3bqX?Ue-Qy0Qd&4MeCddc$!xZ5BBb+ z4H``WU)I)Se!_4Tg9_rj^dY$lj7RqEA{gAv-jiLHQQeY=ijkB#g;$Dfk?dxyG@^g>|Qc)Y*5D|ro^;!u9mc0$|=(L=D? z=1k?k#4NqfC1U<4N>)}VJRNjtP&F)0ZnWy1e|!o(WX>x4&6g7MMtU5f~JRBNpC&_p}23irQL5|b@c~8;#~^`s86-iU-fOi zMcDT9?eW?q`)^iT^F~s# z(2$;BPTy>ckJ`C>hH}-7tG6Q_#HFPlM`^P|j3OUR=E|P`*ngGeT_`mvtqK(S%8sq_ zmi3w#`skg$WdB#=tGs6v@VD{RxM9;7(9Zb$m1T6kjs(~;@bOLgU}^DEto*UG-u8Ou z%Y<%Ki*peCc}u3j-jc8OVQC7 z^5KiM)%`fp^`z^mbr4TyDnh*Gayr_WuK;;i?RT`>hN0JL$Vbwfvs-iW?_J3bW4C(j zF@skwQv?D=kaNQ=JG`#K-Nss!Xsd0J+b=N8Z5~~w`HgQ4wBog7m?0FAkFQTZUvtm~ zv0jci|LNr<`5b`HNjnMMxT0)8!A@s^Q@%oWR4) zwg=$7IA7>xBO>2z7;kK+eKi7r-E~pXRSL9Tra7)q>&npf1)y4ZX~uoQ#2v|!L1Z=u z>v})(T>6T7Zpoo-(0iNp^b4+B+Ff+w#%EsQ@Y4nuLVJrLExzt#`t7%ox^-AX#WkGr zxBa}S@Z1%ug=5&^zb5<|P^CnGj=Rw;c1_9-{OvtoyM}&nUGwAUt=`fhj~2_lFv7ud z=vR_#T5RcUqG)ZFOw(;7lfi`cxE3grp}7uL#f!j>4*UAc#GuV*p9SWBi`>3DdqVTn zHWHy@X{_aoa-q48ljYa#(HnmNO zrBse0xU%JQRFiNA14en&h;j`8wlW zDl-o95)~_@q5-7Y?+gDCf!0Ji_o?Ss5nLfqXymvDlCw}#Ch%Rf!v1n++slz1z_Jgo z*^1vR{LPffhFPc&wUQiJ*xd?!PC(_%lu=oW{@d77n!qN%QhG#-xjI~@muQQkGp<$b z=CqVPS&NFDxBLNVg|>|=&vvqWqq#qEuf8)D3S%4c{siXcTaX|S;;cr)fpv-Q9WDO5 zX`>#rDBKn~L9`W(c1wV0p+6c4F=Wnh>}zg;`9&vSrw=cCU&H(&DsH{$4EHZ;3*S4y zbn7U$TB?D%{A}HD0F=XE>qHABC{3l#tOhnjZ(6p2f*74kt;Ve z@=O<*r>RbixKh${#otFN2nC^=!EliTDsf61Ywb6xsvmhU2Z_8|aHhCwpfox-#fq6} z-L3*3YkSkw=xpeQ_i=lS@3py$eT5IMiiuVp(B-+=w#M!-#Dmkz#}7Xzb7``beXR(J zF;z-^OIjk9K9c|COweCbPLIU=c$u~*G>|h7tE_ngfXLD%Q56_@(?jujnStQk5!oDr z1A^$&e}rRmr)MX%%o}X@Hi4L1G3!2CSrN~q;GbCl-Yl)rbCgtP}(1&Ik7X{$S$La5^d^t?$Sd&EQ*r;$*AwOZfG%gi;H2; zaWj}-My#pybNS8o+J_+-bxS#f0snnt9IpLLmJ4ta&{WhjVFZBPO^$TWYBQi?qNj7v z7P!{C^afMZ$NuV&aRE-?$9Jb?F){wm~zsxio5(uuPwxXec+ze9`DPrjeP0z}Q-bw%#A82ju=C<kNxCI{+$0|!6;&~H|H#KvzNbIug~iRs@|X_=!ViSaJMv$kpQ`$m zHzjigmSccq*I~ysx)PGfV#}`@#mhe+h<+7f{1P5YrTtAM@J(=uW33DN^hQ3VAo~=5 zQe-m4hvGKrOSn-NOPL{UXG780WE_9L-I_I^5s&XJ{(+QgpKmj`Bz?k67zE`PnDD0R zd(*+uDz+%B&NMS={Zqc%qRh7!W!aBKyrij}MX~shHUQ{s&P$$*3tNk~87B(gNm?#j ztm?t5@UfKqhrtn^0n2 z8O?0#EuLHJ0mQQ|ueIiqss*;0FWPbMBdLy109=t64CK@g$rclbKNh^h@OKSGuTL_T zO6S;_Ud(XSV<(N*OAXD5ZP8*yvX&E?9~c;IsKL7F@sNjDa%gy{$)KI-$GNh-a z$;eF;32IAX^v+lf00?@A*!d;Ifgc~-uBou<8cxg<7ebZVkFWziV`)m@=Z4Q?VNtVG z^)Ksl7w9Xf@WQO=VSBW+I%lk8yH3iQ$C}105QPbQ(bw&Dpn4JOq0!Hlacn&EnjNYs9}gTy!)E? z;BT42{H=pG!=t+rW04~#(E-4-&gyua9SjSTZJxipBXSmb&-C??N8KB#p`12-?HkFXLB$(jX=#u)}P;M|g03XGf}AA247)GGSirZbTR?L#UY21Nb5 z`Rp7YRq?HEmhSfE`uXr?PjVp?og4HXLjbBornyY{8wwR8 z;+jb%yhcD(`DzsDN4%#ropn0Nx$a0lw@)0jayv5OUZF>9GDT_WKd*A@i!8lx4n)Io65n3Ud{hSSmR$aRpkHb_V7*A8@Zpt*CHD0Sd1FLS;jsd_Tj%9IqB8) zzHA`apwhm4RP#7oN2ho__XJI!$cVugM04TkXb?8{{q_tFvie2NSxqoT8`q69R(>0C zV8Mn~Fc>-;K~P*p0lV}fGsL!be*!XbkOzrW@+R&gRZ#Q3E7#iLN0s|dH}c`-?(Zyq z^(JPT;=G|d;B*vF4A=21FT9AJW{2$!)6rrznNL9dnJniGsU*}pI0O;Bf^gd`P&7V+ zz*Z3BuwFtU0fPLma-(-if5~^2g{HSV6Y>8389hVw7c!4LhO<>2Z zB|cJU$`vvtZpiI>CRqHm=a@Xq4DI&ZU8w863Hd1VrWfRRytwIDt7o=C8(`bXcA%RG zzp%zDb8K76BS58y;Q2xLLNKf%$~!$Hga-K!%d#r?pAjomG-TWS`YN^&*mhAGX8qFA zOHQ2it%LW7s)}wUoB7E=x=%t)Z4AUSgQFH)ut9CY08NjU6kzysoJuC^|H^@bUL3kA!Gj z9`2gUrc`NvDxyFOW^X9FMSF=bB}F#&qs~9)Jbw-PVkI$WdSQY4?RpyU;&29e%(WOU z1$-dA%z}TM@&La`g*|qF5twH>wBQ3wa@2t=M_23Ye73nc3m1Z$+%>LYS%2AV2_7Tb ziZ_7u0mHMCqdqQgqZs(rzV>EBY1c3^-j1l-mTas%zJASzmTZ=!;DSaos_njki2uF7 z=+&8R?xWGxwhlAe*u)E@diFsxug&$~kzu&QJEVJ9G5WnGhnSwWHL+ku<68*_bo%VD z(vYjTc6w1$Id|)+mVu~Uaw0MYf|kt?`D0FkCH|xd@hi@`36FNUY#*1b1`t|6N=Xidy}$2 z=b#8MawQ)FbP{a@ie8c$yvXIiWm1;)eQr=+)I0ZfG%{aah-m>C)Zw^?P=8qdD^*`z zX8Ap3KEuXTU6x_G{BOO$&ldze)$b2>rk`)ngc4k9Etm6|^!Efx3yGW#Bh7B!o&3-) zL}~P7Wj%Dj+|}MyaS?zMk-!aqb=B)shwXN8FwfZ1=wy<~Jm9I(%Exk#8a$cm_JhD< zo7jT*6w@h+s6g4zQF*ABR;a+BVDix>8(Idf>)pvt5)}nWCsx&Gy-(ZuWzK}|dqe(? zx_3BdkQYJI9(hlxc3+KxBH0Sz4KKtfj5F?y{n(~07nf~@=sYVGq&UDc{yK#IVG zDpipg`lE^pg}K{5JA|3dDBF2vHSVANU|=F5O3?YW|CShE+Mvnc|I|rLU-uPi=0?Kf3P%lXU zrSFzl_c&p*!?#a3e>`z%Ubstxav>&V8+kv0fl7=>_a~9Lw8_~*|4J9G((pKbCJ8pH$aT-u0`NJK^Rq9la-+zsbo_1x~0mM4q zmTk%?Rg)Vz;>mHJ^7>L_@4krdIt@~~+yEy#h8pf&(QbLaI2hxo(ks(aL^hc7h;AV! z6Q0l-d+c6Gy6PI(36B?#=T<-sg7lOE_&}$xNI&axHpZ7{IgTXG76Hp3>--k;$1Nvj z?pjRdrQCt{;SDjzsthlF8@WX0eMn1;p5RaHvPyqdc(E}THUe-@(F9r5=ZDME$_w9~!!7fVeMDO&!lxBN)|+V_Rcj#A z7Dn>3D~Dccn;gG2va_0~GYhq#S zF_-`QrqjP6AZ5;7=9)9He41-w$jz4E0i$=eIH$fp?7XEOpfGGf&igFuoxKCmnTyxI zMtn)@BX!3#KSX#uAY-xE-vZ8O*_K7*E&0|qv1<8+dbs}3bvWO^S0st;flKt;aYi*o zR1Y}ar$XCBfS1TU5e%-Gd_)?b{7-?Jru;J`#A)3|&=7EwUG}1;X>3KVo_k@dA4)gp)Q)Vhvm7?hLA$9n8bH`U^2Fr{Po{5RzhN9pZ?h` z{)(#44bd8YL6*n}jE;+)dH>?dKkg%m!>-5g?^*N6Sg>vF1vJNCa9W`F3}i{O`6P)| z9yYE}HrFxjuiYeRh$|Z6oBEaRIj19`114`I7!^qOy%6no`m5LmbAmHq)JK~d@lWrs z=iA zuO2vLo}_?OT*w`+!FSiFSMcFNz?}*>haInqi6AEo z(18gEn}1K(zL+Y5fXfy>XTR*1`A*+_(j3%70yAh&V8?^~<+YXdt#pHCrppBd_BouU zJ!TW1eYU*c!*Z^M4imPQAz&`cA(Dpa_(m3PV(z)XE}i6rcq^3c)^?`ddsr>ikp@iW|#<$?bZl^68V&aGgvo`zxhc*8nz?xnZ*$yy_ zPZN2p%ea_MqUc@aMvizJ{(D2CWF0fc5~upiye+7txIV#rf>6$TOXJhg{L3yp;wk#| zmw%y|9^YyCYnlMahf+@?DUt`_RBB;3AILY?ob<=)*9!3fhZ2PT+v5|imRgh(D`t#c zUM^0s$=d+y6*Yg|DNtsa0|A%h6@^rH(+il07txJU{EiSiTM$p3YFk5NmYH>Bdp@NR z;&WC!=Ao{NA*6m|#jVJq9Io=of*Dz2jDiLpdI%@T?m#x=$)#3Tz)XOguzFMJcM~|> zMT*gBtJvijl5r<6?mV-S$pd%AN9{Bw>}AOiXV$w5h^KtSJDY4v5)v}&s$W?fBu57( z30-pul!6m=4_MNqe6xQLV@aYu3SMM1bNgsk_MrPVvs+(j-mQcA=sF)IV+CWZOG-|e z<(vZj&AyTw0(~|zGok22Aj7Zbp3h2!qu||1kKgq}1HGN?c@(0*oA^e;<12M+EShqJ z_V`rtz%$71clC1Ftdh6JZ1v{M}vbPw|%l!?f_O%4igQSv30U%Fc6B9(JY=p1Q zbY}tb{P{VXf7GB?^Nmjeu$@Tr9wHax1%mjtTJ>?jBi4(#sCl*TYs}$sFtX}UL@Ydb zWrV84zUC_QqDP>g_bW!ggiuFLw=8Rn?zKjRt((6|XNZq#rjv_$IE`baG5w@;sCV$rbzuoq|$RD=c zMo?DLA8#^VTKczd3k|D-qF3_luwyRUky4f=TpmTs4)^O@`$h2Igo`M_6=Ke!2Nh~T-_-$pFFxhFr>&+QK7 zsIu{U&s6OT$*9V9XnI$g@>l1-MC*?`U(eJBC8HoDL%M(V7T&9}!|<`T$Mcsb&SbDw zT#FG9g7l?bpdAD(2c$k1siSxj`f2}2FuFB(WR1Nwit_}K7}y3kcS`jW^vN7aE&%As z&Niy5Q9AAT*q#fn8hzytY|2Hkg}LUP>Ik!P*dK;;%%tFS@Lh8569-`>#r>;=&leW6 zN$@2+-8SO^)jz{P9lah;UU zX>)pi>IhLFINub#Fozl0%-ocq)Gp8m7GAqlu{+5ePyCFlRL@9+Yp?oN5IRKFHl*FW zMJ_5$I?)-Wd>LahuCdp7z4ji0zA08 zMv79)nNwvD1g#g~T0kiox`^|ki`lDv+VLm*A{aQ^z&>|2X8`geL z?iu8cOe&ojZBQqzyb!g;B^r;wLDcL;R4e-NB*1)i=Nr3YrCFbRXPe#H?^uo7Okk1# z@+84YjOM$$A+e)Nw#DM9iB+rAj-UY5sfC{4y7 zr0uzZ9B11EjFI@z;8K41inh}o!%}_{4g1R{7^phy2T`8_yPMkjC-`lt8)m)cQI?AIkcv7YKV{;R^GFOLeIE5WoH(f0@JJ1v7esy!4 zqvUW-9CJ9-<{C$XfZ%uXvAOeMS}||457DtBzxeyTIJBLOE*(JPnDbg8#*!{>)Xttx z2pDP}bhP#o;@aKpu{KB*A}>Z#|BDbk2p9QRLNx0?B}AP(9|%zngjRYcN_3gpLu0Ym`ElO~0lJL;({uUt5FmEm|xVn}OG zHTlT5DiRF7;n7MiVs3BZ8_ujQo%i}lUQkg^FsSlN+kCJ@JfN|RPxHy1(q27jHT9iZ zhP{Fc@A03~6^O|u*j7b{jo8&8`~?!+R8_VLKWR9bXoK&Cj9vxatIhL6x!1dCxo~%^ zN+k$fU&959n8|bUeS@87CE*{R!~-Dr6=v)e5)rSLU!(;PXum%l)6fWNOZzg5e@qN< zsVOdxf+81OpLai1S(vJMmG?3E5gA>EA)_|!2-pnBAFoT-lgBC@9ciKzSNn(ZUsCV9 z_p#ad;>{VgBMYnSW#tl05XhUbhZ&$i5rMp6cqH)@JG9SL** z8->XU&1dG5QSO(%aY>w&NoPhW+e>dtsL^{Hj->W`uKB2WdeJGxv>)vC_)!L*?5OrS zJ$(h)M{7K`2|$|ev%f4N-XpfL2QEJBt<+t@Y@jyF*&Je|JmzOXw!h-4cp}NNODktK z(mP0Wzj3XL!`12Z)=rb?1k}97`4@Z=6I3NhaBZOv{_KnYZ17i`lH|{KDi~~rFKLws z_|kTEVe2eAjf6lI{Kx@-rMS^LCI)AQ%ZsMwKaMa>D!qS z5$R3B*$wBdVfjMH&eH>{FZ3?EFNiTWf7iJGXc=$TclGNI`mOB%M7Q~|X%8xb64G+A z;~R=2|Q|4Jz@;2Q+S~o=DresX#I$6cd zgqAm0DNCh{H38*5{aHeQvQ3F_wC}aPY|?BgVUH(NxUYmjT0XIhOO7RK^~aOMqY1+> z!&JO6X{pfcXYhVh2$t3}eit-)PE?$~JamT#r8kj|Ivim>;b8l(w-aoc$=Ycz4RDJC^P`TY!nWWzuHGRP}hA=00 zyi%t2h&9Af4k~<^n1Uq(nH0?^7Wt`EIE*5N_0I5&(u-pxJ&$oI#O<=jUX8m3C@H-A z@@S9mHC@B1Ml{IzIjo5#4`Y*@#&3ya`tN_rCla=XyBdo5Y}nbp@R&V!Xi+qe@sh^xqv~@0Hl9ZkkPa{=Pt0` zQ!vah{?HX>LgoJS-p56sFB-Ci>Km{Ns~siz{|aDw{b$)snRi0NfTryER~H9%$d?*v z60Ou^B(8%~TIT8{*SiXtWp-$Ms;_>Z)br;7S|4i^kVJLmGE~%T)CuE4mdP)U_$AzgUkuB_A_BeVTe&+23fG+3h!) zjqBO4W6K6&oCBY+b{o5SGwMmBAUj?@?Fdm* z5Tb)lloC4m9a8!HZJXb)nrps{15bt&J@;_6-APO|G-np;grI%Y7CuuS1sH-vd`$DI zmdl+uTnE3c#TZnQf(TI=q#HC|!HFRS=)3mCp^f zX&!UerM_;S5AfwYBOXsEOtPH#Tj!B0S_m|7=cmNZ2=4%Bl-{Oe+i!q)BHENdE5T;L z=`*Bm+5OwqZ56bZ8861-AF@U-O&P_-SBWeZbk@=|kScbGNLRT9xj)-wrNw+LHHc?y zicO_{>E#V=GuH{Ds*6H=7HTDXdiK4=H)8a%h=>`uKs-K5i4eZ~0!)vsz=d=#Tg{Mz zqKTooZm~;P+OHt(X;uu-p(Bd_0&4JrE2}q$0SjP37_jgieb3@2jdMm+2W;N)f&PAM zS_Yk+7i|zL`$JT0o2Z{~%kt6O0;>B2v23nRS}m&!9J=`Tdq**K2`<2dE+vT-?(s0R4XNh`ox6=CK*Fb+QNLgJ4xB8 zvkMUUj|qbR`0(Nk85p`j!7J0~N8}lU%V&8Q1F*-6&ro3&YB3IJ#>FBa7)w}FZF^&L zk+O_oXyKp8Y84DwT@-fvKbrHP=krzXM1pxAV9>B}3~wo-r*j8|y%jmBqu`=3*0>P> zBb@{gmTI`V*FS6ao<=p#P*jt5<*bHPf5Tih<-Fbgz9H$Vr6S~!pPpbt zKuLveYUw+y%tpd30nR|F5$D@={9z$1PCv2KKTL0mMT{oqZ+8eqJUk?KQooiIbdCOIt^-OarsqMEqtNi+_}?)5iH^i zzmOLB=zZQ#>bvQyrLB|QSR%!1J`Rfr;(~Y3&Jtt&XXR>|nM~5O z5)s5v^@Nx`{I(+#`;zSz%LVU$*v4zRN$Gry7d3BI9cd=&6&^wI7RCqi($@(1HERSmpl_n8?a4jsb(LC#X`lI7W{Tq zEGL-eq4qeE@c;1YT!=epr2hp^?fQeKo=`=l*oSTTKwQ2ykj+O?dOTQp*H5~+U5qC~ zzd;uy*6|;6AJ#v_g*bKWyA#tzU^uMyaz4gcvV{%nTw_J81NY_T@aI8aSfe;n=hGZr z;eLtJMr~0pHiLmLyyP1nmOP9uLE>QdYc84fHq`}B+)_mtBmm=e#*fKUNiZh4GrF@x z)9U~9X)=LA33C>*?caLt1Uuc^ZvGNGb8m>sZVUqyZkO)!bxiKgh0~5eJC|0td7%VK zLVcGA6%_im_Ux&~MBvhf*MhS4D;)~6M1+->PRC;_QX3atH#@MWLgC$3yC}_7I4OS`h$N`oyW4X!=)$CJ6FE`O5Gm9C zC(Zf;gJZz{c>yt2_w%S>Zqgb9k-=f~zJ#(b)E-dXw&7ZG18Nk1f1u4F3;G}qg2#EP zzxx^ay)wc)A-097a{9jpm%Wuf*lwo7djW;NjGQmTw9$TNg%>`yb0lCetbPKE$Z1SO zNTnUYTCF6mexKA4PZf2WCgpzooV^K~E9Z0^;;L)&Y4g!VZcEWyQbP}-B+ZMy;O8W!QS&jWK*25%a!Mi&$~iyA`jm zjtIVt^gqiUGSY&9mtCA%KI#S5=>o5IBuqUhF^bgKkBJs%_>bpkhtga4+ghIzjsNPo zE`ujt(s}(fMg|k@n4Rs{GtsI6d_b>o?5*s^JX!6ZtTg5QWh%=3p#yH&!k3(q40R7> zZRhcd9;U}YcjtR9ssxAT+ROBe2VT(dv?-rA34~blw_H0~V0}*96eiw}d~UOG@G~ zYCRyoLsBTrdWTNvj1QwLmv#uS&-2DU-%B(ya~aiOKveyJva?>BsffGUMGUstCcJb) znE`FE*-k|RCT(ul)Nl)V>HiDtI&>Hoz)8urAc<=Y12QVliD>mvn!U^k&#H|K#%quh>Q;z7=^G&aobc8&wvX_%Bvpy2ig^ zu{{aDphwU7o?qRy?ppUR*K!tnX7=ov{l@cr9t28PK0v-4*N;To zgQ4T5!_F&#-g9lUN+_{>qO7h`XfBho#<2K_LtpWbe=;qmyUpX|vl32+@ej$ModvC3 z9(R(hyoX+NmdD9F%mnW(n$*UIX7ViQ*3Ni+q)K9}2pLCnR^N4J>=oOi

vIM~Tcweg@1kQaWSlNjZU92-BwAs~Bz@&c;X0@(Pbo`cQ5>dy4@PIip z?+9~Sa-~#}#=ohy1$j#9DU~dtL-*#DyXyrLPz+1x6yW=^KP0m<0=81>0UnF>4SXYi zO%d6OYN;7xl4(M$d&;nTEg!aJgv_i&91fyLOc-8klNPYoTN919^V)N^QB*xGEl_JH5y+nMkhsz3aqkg{mCWcdc4`o!@ope>DP z+b8zA0a}?A-xzL(oNVcTDY^vi%m?k+*tkwBPIRyiP-eBGj(`kA3nL#UEyVjU$ds0M zaLhZrdY^P=Fmz%645e01cmLs0!o}$^q0_|fqp@%XvYU-CHv&6;bOw*A&f;(6@-av5 zxw5N0c28cOu<(A`+p2CZL$i8d@xeOgW+xPaDe%k*`jE|YDC~5QG#lO5tH%1NgmQSNHKSF?TA#;-mh9M2U%0Hyuk z%V~vtJU?+!KTd+w$ZpimW^519pdfnW{?h1Z|ER7{xR1ATqHV{2rmCxFC4yjoD@Pwu8M6vyQh|^b$eZkTGvy;9%agyaz}}*tW~CQ=EhQ!K}cq`lar%XRHiH_W;uQ z+>;dZCo-Of1F^IOr#7^`{)|rOMtV2*7>aiO;?Z_Ld=j zY$Fma1LYKx4mZU~Nx3V&X3fL-kW5_iP&w^3w8HtP-@JrYC_Nk1;v6Rd3CUR&*&_|K zv`HtFO6T52mHu`=)(e@$;Ondtt_jn|1}r8}=i3gwS}789p5#T@%pG3={s#PGvuVq_ zdxNDOxwfbkP`=qSGjmt+&*RjoT56dq9-VK4t84c(dF)I>={7DZ=0yIu1X+4@13NuCFx9XGF>I;Y(ZzE}OQ?7Vt(ITE_pptcbv zK;2m6Ny1!8bf$jn-p8tBU(%LF!AexUu3ta|Iw4iQxoCTj-wOYDZP)$%MRaI)t=Bmz zO20O#xP=L_=bhBQ>1sv()YZ!L?VpRt{=b+&t`J=G5M%XRw>y?@zuFyXc98rHk879N zg)yP1MdJ%#{)bbT4k|{Cu}DC>8Z(mE^V<;_Y+<6i`ZFD(*$EydHH;06Ok`vywwb<_ zr`Fh)zT+}w7;fdCzdN@B2qa#x)fB|A5i?E$GNU^Y5EjT1pm#9Re9c3PUgOHW7qon7 zfA9m4y?RX%zjT`OR0^%0nH~*D3J=>@6Xx>4IN|jG#J!(QXE|Y)TAwsmo^gZMu1Z-=Bzq&z#&| zXThinW&j_at7|dLWV{Y#mx!TcvB^#v`?}XDP6Isd2^3vS`QjO-eVp4vbnFadD2H!( zf$e5V+B#Pdv#$TlvC_%r9#qZkGlom7!!4H*ts%o}Y|}Zd9eN0Rhd74@ln6@63p);x zAzE8~;{n?1wM>3RS6L|D-KYH}^YVSd5uDkyyf_nKb^Mi_h_(~N+vQ>Y56$kp#Q&h# z?GveDR7zXcG+lZIQLOBT4^aa!QXwCq--u!m!-UBCcy!VL0zrKp3~A&ZIzn=eAPX$w zz))DVhSBPD?=tH_CKuEx^Mt{iA^yk3 zpE9;F7v?z7fDR;mG`(2$G{l_NAJ-AH+aWd0icCk?ptpUT!?Yf~&2o^}H6Qj=<6+{I zUe)c<&hKk1yVa2`Jv-#T{c~fn2I5SDPxlbl*~fb{>wa}4Y)}0S&`U*~I!Bm34lqLldY%c&B0GgZECWpo#O=g;@_sc!S zoigb$)7n8pWDm0)4B~7Vy{YY&Jrl})Kls^Se2Hg7dOvitp-`;2dk5>1rwahL7FF80;TW7}fZ2TSC;1k^e4;>W;uu*fi@vI?#GR@P|d z>Eex|VCnJmP9)jpp!kc~4#?~)=&)5JALHj(=hq#97))Jo$Cslp!$a&f+>9^9Rn~@9g__UIx z15AwN5;r%+G9XKmLq7=z%bJM^pMDVzyd%dgh>23^A|q_VnNc`xX*&3AW#7hO*;t0B z?iRCFF3m+iT^t^gf+AzTBV>V*EKY4N_&7Mvomy=64s4(HEmJObZZx$ok*1C{hr;-s zc7)4?`z3b zV5KQpKh$NGvu5!Ex8VZrubZLiufTb+{>fbEdAi2OcY({At4q&@KORmWhk7MHdW5n* zTg`^ByNBB#&S2+Gy28qo8~)1liK%asm3WAwqD+|kXDjnPo?(dXw+GH4_^$8tRWN@?(oB9#IOeIt;pHBX~r#g-a zrP_wDp;sSwnGxS)#QjzleQ}0o>itk}Z?7c2&~w~u{m5gGm3D$rg+grT>8+x<&KMY6 zdcGtyL6-}x^cCymhg8QxmMD#pJ+oKZ7HWP>{-r#BYl@}PDDA?H56XRnf;T4zUGP3A zzzd1*WKnh@X+4WFtq0C%_fUt8K$-orUjLm8`N04&kB^KarfZia*r}d&H~oN>+ z(=L(2bGtU^X0t^WTbsykl}Dr=W`Za=rTap^=kymy_aAN;&lTdO;HKsl$k`_pt@d~<8Dpen+&T~YVeUIyB<;)k?)Q9&<0wg{F^`VeZ;{|}WT#?)yhNxWZlCW7h z&pK!DV0-J>+`hoGp_tE9qDFLRdG%Y&j=1rTIz4$)#2!Up8HU;>SS;LM4H)0Jl3D!9 z`-Wo>)T?xlYA&}r?I+T~ar+xNU92LGO_vda(#ksiFn%#{Edj883_loT;W_rji&_3y zZ4vVY@1!!7pxNy$!LTDf*I%Xex?SGxmycX@zena`DAYF>7#qVm*AR+vRuY+Lk{@5} z452tfM8bESBR4N3-NQFr@eK6ROQsdNz8h(3@`bdQRa(_3SM&1# zIFixJOPHWXwmL zlDWKvvDfpx^7eRYp^pZgf6ltq{ukt;{~jew?OMBeNWx|`hfxc%dOpiM7J3DkrA*Im z6Ldh=PUc^&6?k_%(f_T=mf`eH$WsVhh|ZF3_kmQijmY$owD`H{HzT-GSuK5!6tuV( zTf)1f#bU`34G5nB_-bY7nKfpI*WwOuuq#{wj@6`c0sQlKg7JqO^z))L{X;?<7csc% zd9X#)YEE$IBHy*Ls7E|E0if!p2L^Sm97~H=$E48>-2$$k3AejNZgAWCTjMK5xF1a( zWS8^8fk4}-RVoSTg;}nIE^wQ>X3@T1E!<7?fC|9eiC>HC%w8D7E6C=&=azIrRzyP|8e;xv)y>up6kn7f^v2R2u%Fp1guJDfUkve z4?2tp1MWDlS2aL@gjayo7&xaL;={d&y;j_MF3r9f{|3MQ8-TFYMJvt&)ttrGuoK?qM+?9%gF?AW9N>~xd1q+An*<-1qX+`!_gM};w9pe zLk|2-?@Y8St?Tz7w6{hvnG_v|u*8GG3;vkJJAQoF;ig z5u<$1v69_{3a*hj3OKbmV7vXqc=Zau9I%Q3*NNcB|IKJ1Vl?N()hl%D5(!^reN1QB zNnP{!q=Ov;LlqBAJAr@cZlxC?kPTfCRif>MU%vtSSR7*z3i9{~ufyNK9r5AWMO{th zu&m2zteaq;&u}VrpLi`S1b7edE8P?RF8Th=oAM8ixZuqy^-eVdeTWa+i6ICV^C(s? z)f1pv6#)*(b#hlVsRw>^(Wo)*4js@}>kLwg-62NGeYh3F=OEJ@2H~miCJyw{Qlyi^ zp5s0lJVy1DrLwf$E!B*e`-Ube(Ctq%5})v~dY2MXw173&O!z?$6=%|(qv<{CX!3Y6 zf!X)5xD-0QtrYz2S<5MC#;?(n=wIibl2klk`j&NDwy!04xrn-j0r+~y0Om3Ec4a8G;J;HG-pF3dRCWg?r@4e@RbZ( ztc9yW%U`(RbOJSBpT#z42X{fSVUNtW_q3}RVr-BhxvD}vBjf@*<4*{cWXYlB9PZ)$=S)l;XNSd< z7jSJ8`!_R&XG>yOTn9#u)%%2=T)Rj?#iztTe6`N3A7LgnbwTr0sq*TXirwtU)_c%=oprEY`m-G9!{ z&$aJ@@Y4DiMk<_hHawP>bf4|~;e%aF$eh-Nzcc?=)z{A_IPBF4j-fF`Cw2~Q>_OkR z7qyl=>q$%JAc% z8pA9#c9D)DV5q8pSgc#p{vakYAL)+0>HhgDIFK>Ft;{Rw+ z_tjzUxH*zgM*4?9KWF)ZG~~9_sUEZh?2DC@{J1N(Vq$4>0vao^3&354OLU=t;ZHj%N#v z?jqOIB)#Ynm(NnKnrD;+vuVrNG0aDv04RE~cfPPf(jDtwJ5=7*S`c(F>V1)#5J=%P%36Zyig}0r z#z@c&>RUy1P`)ElfsNc5O<)nby9JQ?FsnRKZtu@o?P%JFA!~F6jqq#(cgE?tC%A-E zJDRaLX;$J{T=+=CPgw5^)P5p51Zjyaet@=*;}+ntlgW2up$Ps3PhKoeLp)E}h?R5u zG!f~v4BiExV`Biv#!_BVkp~vYohoHx(3>&EAcC%ol2QkqPq=t||Hp&30m$!nZh5UP zX`H7NEdL?$jo?$p0Fef`uaf zXlG8XL}nP*P}=Jg_903=98d<;Ef9E3da>6#jfR6%1KkFsSFMd3{IwxK3oWL<8?4wm zZ)REZ=!EMc3+;4|{Gj~OXsNrmgLh?zm+R$~2YN6EqZRIfrV@gsSV7!;8G!0&63}sF$QRcn zdztWMEmj-Qf02|_n(yE!sqdsEq* zscBA~Yq*&SV>G16cLt8Z34#-xS8lLTk?!5e0R0QVIe1dxHhcIL$WS2!jP2aNzYC&u zzwj`LP*n7dLZB^G)%~7q?I!E__6NxlR}Ml9n%zGD&KfyOwScGFK$p925P$QiXM}X| z9d18`Rbaxd({)2HGVvz9Iz%i9#c$t>;>%$7A<6jw@$1$#ecA+frj4_N*QEDE@d02> z>E%IM&eLi5E^zOyb{NW)lyYpb$tMD(8_`2~-!^=8yqHM-5rtWr>!IsA@PU@Zj|N$L zLelp@o6<)t@a_jydevt2}`MtSAkTGxuPzbd9I^ zuxW5!2&ERcR%k}1MF|Zj$y$8wtXZ0+glJ@%!!GgRdcDJa7k2h>i?P_$a@gDA>_-ys zl#7g3*IXNI)>FNQP<<=B?yx?|T*VO)&@M_JzE{G*g7 z9aq+M9NW=}GF<+EsXTk6Pq5F{8x)!;;?zvNO^^dEy622ztFS&$4`O`Q`C!E(`Z4)r z6r9L^@}yru$cc`0lB*6d+w_RNmkZm2gO_^90lz(nb?VVOGkkW5PcP@#QXAFC&HY_*mp!&`oCP@%EHW#B_``}~Q-u_QQrLUD zgf+cATdaslRf_~SzGTVVF5$u!&)E7Z_pa+uEsFt@qmWKWw~%g3bN_i`t%NKmASP6( zrKURm(7?*_p-yCMI(cj)-yx?}{~(T&^RC;h)_clSaxmOGl0{nsZID;!gV*3D0^kL2Bh~A-7k+IUuokS-ntEP#Vmq(jU^!*phWI zVp{0lOUGNL82`<9f5WRvD=X@O8}FpDf8?g7I1Bg9$=}tmOb4O_ffas8T@a2 z23>#p41E6Z8Nm8)$i`nY0st`zZ@-PL&bJza%cgbNT3=*TOk0KW+R1b6g$7!|7Q?`? zhCJ!mYrSA=R|;0P%jecp;|STy2qSUBs0p{E$HHaeTr1KQ-DTzNQ<2lR;mB&u8G|{R zK6Vem^}q#8L65i478}*@w$9n*7J+ElH*6z-4=ubIM4n_f<8}79>?kT@(6jegd%Z5? zP;1Oh@A75Tmf0F3;85T}9Ew1JqpYWxWlY;g{5V<>kEZcB%JattYa1UXkZYon)<>_8 z$@0q6_1JP!&P+Yzp8&=}WA&nO+4P3iVF_Z9l$?X6Xlu32*)((j(6wq#bmYOI$G9f` zL1(bH*JHq(()nWFhWz2#7?Za7LEDH2>S09P1BoB{L*I6Jl7+S(Vj%(82!LVzmycca z#Ma%D&&O9|aOiF9Y5M}JYwXke6RT8C004l&Uc!>U;5D3Iy-q4i+)4HeU+vyb8f##x z`~QLKV0`-jmFqxkeqP{V0`Y(EIv~u5=WqVWpAK5K%Gi0GdS5#1inC*zhq9fa-Y+0y zuSsPmz*CfsIln`&a{2=2nscqdW<%uX56l?tt`>US9b9hQ=WEBJ_Kmiw0v=yv11TwJ z5I`0Zx>KZkI$SN-o+qC4B7Z$dBMx+Xu$zm|76^}(8)RYt;bFz*w4de~u#YTB`MrAE zaEC{w+^{lwFngE>=*dF3Pxm7L<)Scp>zaDw8@$QUhPSsaT&Sg}!B^oYlSNKDj$LTU zgxqOXiIx(N_Ra2wt>XyViHM!Jh*G?C{8$Xnaw;CBEgpf;3xh{h#d;xfN8y(4k7CWb zpU&E7%-2~XomK_!EynttLUXpFjk52N2!I{)E$bWnJT-^mQ=YI}3Fa|1+=4^APf z_1*}txGlO|caz&rtKSr1CfKCK4CbDCy~MHLl90vk8Ep%Am#jA zJdt<#E6s#n86An(M5tcF3ST=FBHovIncFXw-Q99T9T-0^^jqwQ>6pW*Dbjr+AF=_J z3b>5}*IQmX{IcK<>4(tL^?2Y>L--hbkqqLDUqHs>fzkejfq>)uuf-W7rE*l7|4s~< z2~pKZ((?@046pugq~pajRGi&fI#D(8>6FD7``uvB&bz^a&grKUAosqy6Lb)0Tj@G<8fg4-+CNSlw5YEmHi)2 z=s`LF6XwK?Gwlcg&btq6jPs4MWqGHKL!wCe;~C3s?H2yJ zYgLQjp!G$IXX&QCf#ydQCSU0k5oMM6!z^C&rxpoL4YU~{_ggWLVV!pWpbQP>Jin^2 z5o+o{->RUpuOR%8+K05nZB)|bc=Jn0^r&&FZ+j{^s;Ln%4+~Ys5cwz`g{%w zl%kg@_^-5ZHjjEsVa8mIn0J=IQnLti{R`l zEfeRDU3BsVdUL7376b@8dBSPCEV0qi9&62+p;>8~;^)G%gI6DQM?J!)oG7$f`6DQw ze^r&WpMtUE4Oi=L$Ruj$%28Wn=I)-f0nG+Gz$OtFO~+z%dw~zH&cJ~5X?{ZC89=ox48Ji_`v7zfaWaTlw%l#+w7p%Vu-+axxk7yCd z%-p0VTtWuAa^Ri}j-Q^4ZaVn`pyH&?X9;?}^qM^^rl>h#+TpWFmTR_O*{<@jXC2-^ zmkY4s0x;#h;y%{dq7kLl`hsQjL<5EpZ~JE=1z*%9PC>+$v)W;rQ~RYam{(|DLi-J4 z2C>utzaTQgr6Itl_05c5zn^Uxccqjwz2+)Wl+HV?s{v-;x3=~pLz!k!CzzLD2OD?c zVX6b`4lh4m5X>!B-foY}Q&-2YU`F@T?gGi8fFnFf#Y8JKBHi^;>Lme5pivav>62?# zxrE5<+4(`_U4SVJDK)S9g2idlZm9jr*D5QHpfT)?NAO{Jm<9cu9$-He`{ILZ_77Hh z?g83Ig@Xb!71GH&2UOOJ1A@b$R_J7(VCl9Oyk@ouuF>%4d`^IFT)U@t9d`0y&qvpf zB1j*OT`yR=t|@O0|H3@lHp&oWt256z+h(~gPaJlGN`@qmiZ~X)d$u`0364U9i4T4P z9A^%=VWW1RY1Kbmmz-yVg(}k=WVV{A{>t>yo2e~ErZX4x-}0-jpJw+8&=mj5-e4`1aQ3 z44rRV&&*-$aWC`hu766pmZ;V0IUFo#_g{0KK7EK446n#zv&m-p{zPlc%TMu0i-UK< z9yBs@LJ}mnXH#kQYx#wouSZgf=n#hu_gyhI2XJhW9jME&NCUvpK>uQ_kO@$Yb`3l;M19G$u16w;p*6=0j zqm8$lx3d9KsxhQz>AV1KiVVT`WPM)Sky-Qo0g795E_!D1LvfR?pH~Rm^E8( zYH~~Vpf}qVV7Oq|w|rgOuUKZ6r*1V21zVA({)En4gVO8i{{*GaYyKUS9zDh3Vr24} zorB=u;oSm)g&&-EM9u(!is`*b@5`gNDFk20q758On59QNLd6q^K%M$V7c|^Vh0TK6 zQV6rc#uaed`p<&^^b3EDs>|;g-?KEY?JV`}#TBj?_ zJ||2-5IlQz+TM$)3(TyuGwLq}-Vk^4c0D!eN1!_JT?I0w@Zf`Fp2h(f!YWG{En zm7(zf5wNXv?QuB9jfofpceZ(C`qh;(JK<^_ElO~k4d3b!6Vld=rplIn7%I2BM?|Nw z=Jv=De|w&*rMp^)5c+%6%pMbQO^a$gd5nR+UUBt}@I$;fD(w-IJS}7cs?oOnnd@EF zP^Zp>>C7de>1ur7Z>SI#Kin7O8{rmC&)1_60B`dKxK ze*VI*53dI9oO}AWjM}G}?1?xEoe|t%mdhtUap+y)vTx!x5D71w5Mz9ADOvZtRfeek zp0*>J_a+?Q!oayi1@87;j-TW`6}gt)(y6?5XxxoEyruER`dO7FwK+BFlJkK!0cvN%{fCTpj6`$C>EN1Sc880HayX^TF}>$JAJj*)`zqdcm&Y73HgSS1J0>`v9ECX7y=p?*C?I=PNE@x-Zf|8` z<{3Pq57V#Jy%ruQpc~x+XM|Ag7`(u;-s-?XS)-^RD{5ZP=h5E((fo{-zMIHaA~V4o zJl8nu-7jEv^kZgs>%sU38je02k~siniS<^*Rqt(T1cl7)Amzp{ys*F@vXo=6PnorZ z6Xl0QyVWZqJ2w6qrZY^$+7|^~zFi5Cp6u3-t|sHipuPJqCr}F->UjF*8e~JrfnYI_ z4JLIaABg?XI2cb3!?oW4GGBfy1*0jI)dV_i+F@WkE<)gle-~9ZI1&^qIvjgetF{^b z7jAv(p$tp7GDUyg9js7nB50^6VuLcv>LNxTvFX9x(~mQ>{=uQ1C7fX0kQ#J;kDSH4xhDqnN0Nyq<&=}9glLs=0{o4#Atw*e>9 zu#-IqOd)z1xp?YgX?B_$yd z!XZ@4x;(#O`0>eO~J}$3Iri7dL3AXOSr0gF+qVF2^Y>|wcw0> zcu7p6cK_qY5sQx#hxu^hv6ugE<8f#9M<8`j-fI`0I;aJ(Yr1#!F0HjrHe89-o}kj+ z6!xXMRaUVj0ayv?pllx#FmJkNubGn6}(o0a0oM z+97Zwfr%irr=X_zlSFS|6G%xXhV=bnyH~R{wclnnP=Rn;0o60;8@zh1$G(Sf#IY%^ zVuX;e$^RN7pgj4X)ibmr&%7o}2U~|;%;J!@Ef5Uu9?08<1Y_Htb;?hX+#6g3!pWN5 znYi@d>`ds2Y|8wdaCuPn#>=}c z*eQeq!>qg1B-u9lhgl`0fA2*!(3;PBcc;rD;`PaeUncXuxxHx*h}+>-4yX0^N_Q3H z&3jxrV)|2yymUbO`jX1NK6CoC0vA7SVW6K1W>^Ljfu)C>gP|W?<*-guZ0lwUh*bE4 zr$-2KoF6Gydc6iFNy#kSh@%fV#*KprT)uzOH1-;`?RA#2gO~1muUhd79-WvnFbISD zo^KMZY%S%E{;t(W9-5VIvgtb*#3Y*10f@3^%p{hpdG9cc0hgiKCVfc!(vA^ZkEQ{$B;SUnZAx^+xHhkOe;8O7nyhm zd(O9Btfiw#jo7Hs9C^SCeY+3vLf>T7Aw3&2b&(7Ow32MpPB%a@(9vhx_$8b@a{;jl z!VR;m7;Q8jHbu%nz_l@<-L4Z3NM*p7^X`Ww0(1T7Jz^UMlk#umoyFKOsGi|$DJ#ba z$NW$MjcftWejC6daG}bz^hr=4x#SBkNO)Hq#@JA}zh~>ZVyAX!b7%6Aya?|^U?YY* zP~oqJS>vRF1qb|26PQcKHpkk>zI2Dl)_7{2+}=3^cYoM0Q(G&S_9Q_LEUQ?N^muL( z<1614J8?7G2814CWSXz$>rjp+(dgy1bF;RNIc+!b&2ab;mlghbw{VuRojuVctJLXr%x`llnb0#tumxQ#rWFS0(?#$xwopMxwrO^+P^xOkA;I z3FD^ZUB2>mP1o3DUvk4>`JAmbu{WpAw_HKLvK7PcA7t9bChxRpy2Da{GWO*1E40qR zPVGI@9E$(0lCSL^m2V?o=YSWlnSf=DMM?2CiM!R&O{|;n`S7K#agSW+?Z6bVf?dT9 zKJ8-+hXp+ssyv5U&{K0)k&K-KEv}bNL-19oMqcq9(J$GpL+PWb4CWW!{HXW&rPUU* z2OiTkxv{L?yYu^y!%>Y`JKM3!NTSFhZ;e_`H-}d9hOwT+0c}a|>$HCk5ZqE2cL}T;r zbl?^${dxP8v;<#v4D|SkFUpt(GsnHqoICj8zN=>6nq438_=t?Lb|A*~55 zJ@Q9x5RP?4u1R(6n}rj72Zy8$*qZ3>7oO1gRauqUr$L3eQGQU&R7?oGR|9?WT(yLYumpLM(1FlCdHG*N7y0PNyP(14iE_k$7(; zsU=Y5yvO89B9vb}x7rcRtVr)>mY3}|xf=kaor?_mUIju44gJKy+c$=qOd;zFD)|23 zcb!xYtMosN+hoN{)AM+*vEJEgCoMvCof>YjDm;5PEugG>eUL6;pHX4T8030CRfkw% ze;1fTyy)d8P%8?&qV+&A@lnJ8Fubq3VTL2f7E-+!-oIKA5oSFG4FLCyK6?La)O~7` zV+Ox1TL{dCb{HA%>m0&CrJg@-@b2vgJ&#(a)`jzRwkh1Iq-Vi8&|7y7R=nF%ZK!~r zq?z%M1Twb~)d7B%+EIm6%WpZ0nW`UhFT1lXmL;{vZ97rrcv%-d$G+g{k|k2r>5fKtwo$?WQYyhT)n+{PEO_SH{lwE@nVqNuX#2 zX8lCoOdWgJRI;CRe0%G)XMSZXp|FB~j+%MYdC8AXjBEOVVwpP?fUiRo@tEAO`>2;Q z=!Pg2S{ps+rjS?3RxGiXenjbV6FBMo42X){Z!|I*H}#rZ`Q^6P1o9>q_+d{sFG{CO z36i7Urz6pK$~Jd%9Dcz;{!cSUCWK0rqPPhCVvk%P6#60Dy70-Z2sEC$^$!n;ut`sD zbn+7Eff&B87^hLrt+%#Re6!Nx=j02T#a&L+@gx#uS?!MWv1t$jt3KaGesG{oVp8Jb zKV7ZdFV>Qb3_lGkLqtmBSoY zM~j<94$p``Und#w(ofi2rtYdFXTvx_;l7H|HdWfca)&FLUjcTSuII>y*K=f^!N7hE z!t*1cxIdwcw@Kx)XCMEtX8u;F!B0}u2E_efc~zrGJCcQbxj)wpueJqi=FIrPV5j>> zI@~5gt}1yi6N~C;C|!wx94anDP_E8mEK|Dxdp6@E7IXVAsAn~r^k#hPZe*)9xvBul ze3*v!?}#0D^CIk#Ux*z5d{1WYX3*7JK|Y)0aDO@LRd3tjIHv}0$i_-(%{=H>2*mDk zsyzTb$z~`H3?TLQ_9DKjc`?^*??%)OPq}Vr+v=${r)Qu)Y>X}Q!1)uvBO@%WD_OR> zbu|p;aLLM)P`KIS17t`3P?VZoWCk48ZBdnH_vk_E?-Xatb%89HWF#RX(XKGtTfU7I zTLHUH3NnP@eNT%6=?H;RQj&KwR{4j&)l%w*{O@I@`niK&x@~QLWNs)Eh<>sALMSdd zrYCL9^PN?(Vd#0yEL-A5{Uz|MClUizU9Y4s2a36dM+3lQgV2qYNjXDBGHJB^gn;zh zz-G{e@C1kfp~0#ylq?TXSuc}M751F__A{KazU-zN5O$rdlA8sG7mD_voy+uxwhk3q z0At`O9K)dZyEn61LnBueNWd-gPS{-H20GaCYyeS>S(XZP($N?1%g+#ktVKn5_@YWF zBlGf`jQeRP5-}lLN-@X8f|$!VuVp{^Bhmf@uvm>BD``a6hI99-hr~TBzaVcxL}Mz!4M{IiAWecOQ$AIF=$i2J zw~$IHB?jw!@kZnM)_{{ZugIpR>82becd z9O%~jy^B?dxHmu%{>xSD^6FlJZyG$6r;z0Mw%u#eXqp!Fdp6dK-++MBIRhNl-*_iT1uDtgfFd_85LJwTfHwFvl`5yGsBFuSEf}(6N6Jko1D0JqFO2&RoVhtVMObpDy#m9~D!=_-@ec zjKG+=77!~uuzid_<-h|xAT6adyXd3^yZHC#HGBR6%&LU%f4qV=y@CC3BnACbH7mr4t&?(ykN^drK`4eK$+Uh6^ zj*-UOc?vE!n`Xyi>`HXJfdaT*KYc*tkRO(n%9OED6c<=d{)q9xnj|ho=au;U>%9hc z&vFRNpVKz0@n)q5=G*`oTw?B=Ue%^s$|8+EzIdLA+KP4xJP{UFrMKXt!PT<r z_A)b3)Kl|rV1xh<&;tJGBzWEJe6boWlL!#L4Een4Q7tXz63&Jr{EM@xcXZ!M9&XtL z=qA0t#Sf1`(2;#HhXN#zHf63nHF$4M_6+zxSV}?@{6`%_8%Q4AYBOnj3_otbWK-(B zY!lM8v8Ol%<34yM-8E+=)jI@$*&$sIkg{tk-ifGF>zfgR8b#x1{ZYfVi3YS^*Ra)x zE5PA@&;oUAgi!qxY}E3|h(xfPgG+Q9JVive>I_( zP^w-B|21>nY=c~LhkiOofcONm86EHMBf%mAC|V8ttLycFOc(I0&5xGH;=JlUpEJV) zXFBr}_VgIRoFU*sF@2t1)~1`Vo8;=KeTsJ#xPT~XG*|twSE0CX*7^|08WM?U{udj} z&#>|P9^PImq;`1S@rkcrKY@R50)i4&j)|&b$gP%%)$jy^ozc=jYj+JV9JIZz4tDP$ zJB;tMQ5K~(l1GzA1Ikvc4Z+n_(jo#Zn;vY;imk1m1jIHRDSL!i37;asuhWv8x^~^e zUjO4X)gC~XgvT1f2KNVqa;O#a4+#!3xjwIq4^Kx5 zA!EtA5kFO)Q#s@{Wzc#rja8(kT)@x_q+nD49m|V*@T7sS6C4(=kO_mpV!Bp-ZN|vVJ&Qfo*|{T}iS zW>xpnZGn&2XDaFaBK)WG^GZ>(knOD<$Cp3Sy#NLob#K@ZghrhDXw4ASaY}`eQOLsR z>1aJ087iyCo|Us2gt>ZLmxmFV_`It9b_*t?2__l-@^04eb}H^f;JqmmuSlf#U%9!f zD7d=+ut{h%lW&Tff2bFit?PrD14sTN|QTEeg&) zRL`zL^U%~4^46gluywI<0`>1*DRgyF?I7CV;$k|HM^|3`Fms604E>bP`-ftZ?|>m) zh_&)m;5W!)`qqq}316cDR^9D&jaGR;f^Yx1mQ)*E@Uv50_fb~b%B|)Ou;SY)7BE=R zXYsDjWr`d$w-}!3^$?W7QC7_*m1e1_@F8ZSiaNe#`^DXJc)vHP=X~^Hi>ac}O<&|h z>GCh3k?Tni=j&&dDeOiKbZ%EdBeGK2cI`K86LS6KN&RdDTOu?HZvs+8h(DVUUy)9* z4~fDF0>+@a;uh?iW{i(J+{%#~QYL)VvS04aO(-PzevaQuu1{sKp>PI?&o1)8gMgva z8Uuw5zhGc23LA&U8V-x!u9qBu3{QyqS`h|BMvCLVPp% zMAl=jyr;xcQm@B#+FlQGmc0}2YG6GjJ?<4EzTIUasxb$t3AZ0PqjJvGL@d21ZqDMw z`b7;GE@xsCY6mN{PCq%7^$1+BU-(g)E)tb7km`URK4g?0Li2ZZ6o5i|MxeYz@YAKB zj=XS>d9Qb3kJY;P*tbJC0+;rCOWB2VWyk3Y!8KsXGRX1Q)+v@G->;E*|5BH3<6i46 zAYQ#e!yjr%vp02PrNW@@X6D~ZLz zU9LtCgfb5@ZiUo1@9%Tfvms~|Q4jJRr}1RTYnt^Rvx&`BGd+4f6((i7gL|_|0a2|y zCSLySDAG*M0@unlI{%`35Q+Ix+3*5Z?BVC}0@#nTF4;Q{|`ft+( zXxXYra-TO`gbjZ5_`5lQ%7Nh(&=CW%jj}uI2r_gc6Y9XqsZmhC;!zl3E-3&72t_Vj zZp6CMKPiRISkKZ{1@JjO;-f>a|RCQ)gDqUjOc)fPs%e{EcVdSs6t? znuqHP`RJbEa%vgdKZwlf4A1Ut9gZTQB)khqxhAJd$ke0JP0OH7+~i^Dw&fy)j3|jt zv5TWe?v~%XL&xg5JW5nTGiYAIJ9gp|_louKPj?9I#yXd|~e;WZb(#RHe zLlG(Q>&dvtY?s*qkF@#3fw;e95glptqsha}-2%b64J>q-Z{)mIdo!^qW{tMN6t~pg z1f;`X#RkZU;2~}JB6JCu%3L53G800r7l&dD`fCjhD4I7p*eE&^aq0Lwxg=_>!3xES zZP&fI>(pI;gihGCM07aZN9;*whk-| zGg$PzgXUM_EKlye=Z!w%Y}#+*B>F90~*HBJKb=k1ZoaVBX|M85~mBSfDAnhC={}y4mhglbkMYvF`rg7;j zxNL4c7DxR)AL@Vm++#_yK@|zu%Wrv?muyrw7Qoj**}K&K_Y3SvQqP}sRrEJT@Uv~> z(o!~_eVZ0fDZ8gLu?u1AHgP-W@xDJ*c7<2?s&0~ge)=|GP(O05^BbWTb)4;t_~l6A z>)ac(7~?6U`wP!FU!#O6qK|8BlO1G|O$xNF*Rqy4fjKb_%#*nclCgF53UvokUvANP zf>2j}ya{wl0WZ|Qw{M5RgM7akJ;tBjQdHBnf-&vRMv}cMYuUg~zRDJu9R&AD?@I(9 z_EM(8sJg>|wG6?5sN9=bk-EpcZepmX-?spfNcM{-ud!DW8ZilhRGBH0h{|l@&?AdB zXr~VJpwaOvQ9yo~#{E8y&O~7KaTJ!Uz-`;cwO8Le6}UE}lsZc>AV>Xi^ZfIaL9c)$ z!`=d53T{tx$NMUA&F_j$%l%TT;-~J5Y>o6`AkAHi^ZA78+xtqHulA5-)5c>GP1)W% z%;L zu^Cm3g<@>y4jGn_tnjcpDz4W!gFhBBVWr0*k8#JJtd)iq1gx$Za+nkUZe{sXk_IDv zf%zmx!ITd=!^em@)MMzN;~L+3BJ(kN7SP1vY+d)Mw;|WMvVUn|Z(rx?0&|rhJHWf$ zEbdB8c}E!D@DOnK6D$9ud`-XpMfr+`!xS<-Ixb3(oTaB{#!K@e67ccDLrD`JgXdDUUDdER zEmZU&KDt7my=P-vEQ_yFE6`Livk9;wO+kt`X_yDLH70I;EexWGSNS0RbT0B4z;zpf zJq*8OpLK?{rkqNwSnp*_OuP6~ETbgJ83R|ZCXGj>=>8f19^*cSYtm=q?*?u5zUDw$ zrA@OAe#vz9>0;<~BLBDPRmUK_`UkgCCz^x&*fWL^P5mKN0W1O6Q|lv;zy$vVIuVuh z*4nv*r0>OeeLl9Gn!!yCLdX3|g@2n|l_i;H04t!}GpnU)MAu12G^JtnrL!WpZVZsh zCFB~ge_VKsF@D44HTnma7ZGs`aCw1ffK%LWdhz}`H+{@EXw6$-HjWO~rSuds($D1w z23OqYY8PtzL(*UWnl^RxyWHe+5UhoCCi@Hj7p)l?S&BzM1h(qJe>f;QB(lP=@<3Q6 z`zi>2fN{gz7|LexpQs}&2`v6#|qbmO}dI(Tj1(@3Zp=|dxBt($xV3Ui0-k;#FOvXsX!#z#fLQin5S4H>NjZ5-ci2hWCphM7jmiyGL z?t0b|fJ&;gsGIz}I+OPq1DNgtXn+W6H3fUao6zT*tdwu8)r3(a_RiT-J&ymTFHb!K zdJOWOWk4t*@RL2}Kjh_o!WL&jxT_CSZh)rxm8x}XCh;mvIU7B#Qq;d_R)3vU3>?wV zmI9*);SIEcgDsM$VhRBy=&8+G@s!SkoncMJ2>G4Xf27KiQBA&L z5NZJ7EBqtK_lplKQH;$hx8_tZH-xJ|<*o08>)*!0_~*ow8$CES=%0GYw~wjm_?&nkURDFRQd2wCy1V@=+Cbp2@QFfl=>JTgC}Q zfB4)Eybd#5_tK#RJp2Sad~j8lbn!{*y>wStGt_lY68WGo_lSe9E+Mi;Oo4d0OG?BS0~{?l{K(}65^(>z(Z zM=F_*bN0J|ZUyFUb-~Gcgnj@#VOg=@bZhxtSUVY@Bd7R$5xjj+(l7?cbOjg$!^?p{ zhvN`_d{)5T8|ZvG8kWiE^kmB9?3iiA$b$QWYz#gi2k#9CVIFS-H)Rt~K#kh3fDz0n z*%;~@oj4;7a8aPd`r)Ru8hF!Vko#K@#`#;_l)i}q7_VRTm;)54?^Z*4eBPSmD3Zs) z4(ymp!>N0EZ9^=qr@udoN_XUEkU0$3J`sfyP{l=c%P1mk!h(-#qhw!PY47M$u&?b@ zJeUrWh%|e!bifZDO=0`YSk(<*7mHSbdnp1X0>IUFAjKI>QJ+fc%y;+I_%Cirk(sZ2q{$hYt5vCThcWFqSS z)*_&th6QDsSx?@RR04=|1HSf4b5>^pR{4ne-zUbVB3Oo^)~j zE`Xooy;?$eq|HYCa$?x(m!$KxW5pZo{K!vd?h}|8TUJ|lU%N7DMmsr{zTC+eNtaYy zT$%G5+3Scg(ol!bNeOM+-CpY^Fb201;YRLwa5;d*!S|)Z^Ji zcqQI+Bn^5xVcuGRm;`Mr`}$o_MU*vS+NL|BBzD+QbS@gRhc$K5{u(@I@7UDexPbyX z2W95g#{WexP~m$h&q>mR0nM#}j&bAT9PUsDQ}w)P(z!eR@`heekEfifoK$$#vCDsH zDmh=1fPLRS06`JG<%FRh!`*@9zc81ht|?)4t3;==vg*Dd%7wKf(t7$x;v=K3E~r#? zUJDS7<1%;-S!aB-m(=a8#R$8<`Aw*tfirVYZcq!>-npsnoD)sSpR+$Qj3P(nuy!N~ z$tc+|U}nZS-?RU0XT@`}+>c5d0?wuEp%y1I`I+o(;{~M5$y&N@2BYO3L}omcwc_%9 zG2Z9%_EeE(?}^sd%(xg-r?#e%y?oT%iI>LG(*vZ7F~xd+3;VcYf#U?a=W!OzPn?O| zKcs(n^GQ768SE+}9s>Sq;a?|DR>iDn+#oV?dW1v;U!z;gHyg)4u% zs_|v<);7XbfL6LXe;f2vh5{!-v*B3qk{UPjL)oSD++ccdMh=uTz{eH`?ZJbm;g+StN7nFZj%S+L?OKHCq={Wq_qt9$x zex}P21ZsHG^0QbQT|muVR9?u=$(yVvQouqhYU?b^I)NhWl~%v}%e9~!Kj70&(46yPMoQF+7ji8r@rSWx zMJOYl$%eod^ZLE1GHWXS=iiNF<3ipCP}%XvSNF%2>VO5tnh$Sj)||DJ)7fSl!fqeY zIj}I1pI3`U;Y_exGS9tj(<$B*7dbMyY*ngEud&uDEj8oSAig+*eYH*WC@n46)Qprc zg1#pbJQ5gLD@cyru$GEU`}vX?K^`1Wvb!8tU^6i)c66`WNjKzhbCC9B?()RrlH#j* zrq-82J*9!MZol$ZF%a%abW!^`rqzDj)L6BT_WKmF!@@#<80q7V>@KtU_j&d5_fLN% z#cTym4p(GP33B2|E5rM(%X=;yFgW-~Wb;k?@S}a2Oq-*P=PcW9L)!Gy(gTLqv4${| z41EK=qdeU%Z56wOknqK{Dw6URvuCqzCw;hm7rP1t4I^LIme;sizeVs}GiW~7R^%01%?{jXs3<;P+({9J>7sMr^T*vQ;i0h|3 zuLKSjko0`!7-Z`tX4vU$`?RW?ks5tmUs%>R{OrlyZd=JJ;&<^nHS(6-vy|Y1HQ5E& zr^K)4CiVM-zov8ET^=s5`l$$8gsdapsC*}3MckErw>$0=rw5LTU!$$?h#XOI<5&tv z4RSZ=e}gQw&@B#d%$WvzYBYCC>U=<@Bdjr@QeSdj&%r4%osYt$uT|qiRDVJj-*SmK zMj)n-mY#rYjYB9aq2^^pxcyif_r8j24fKinm8Fl#+m}q_imOeQ%q7OC%C*?bgYrpW zpvSMzowC6-E5sjgRf}9+NZHR#d#2Bo{tWrF2kq#gz>TqRm#~&U2=93N%^sVtj$URU z!AGAqv>J1&R=x2hA9&Xrz!!`JMcW~b`8P1zeuDvg!BHsDI{;sR3E&H;qO_YIX_-EK zSuM&6;0t<-s8Qm=-!ui;aJjf`VprC zm{Y5OXuNX5#Hp)$3ki0YYuqBsWE5AXnyv6+O-qibQJ^;9esr{qjBDxwFU-9E@Kb70 zD{}H)4vAi41=Vh&9B3+5co7hdn**Y8`CYl@FXx(AFTecK?$7hp$UglMZva86M5k=$ z>G2G>VM1~r5rzue%U#lx-n%g7sH=h6_}1fu<`a`PDg4MV-Wj6Z?=mlGZ{VO#Z!|8W zT>-aAQ{}@HPN4lg7`JD-oD73Qwgvxe`yJ4ve`p^Fo?6cZ8<=$Sjg_b6L-C_J0(9RU zzr|Qm=L1(+E@kcI?EhK zyw>u;NRArik`o>ZJ2-D!_OklTi+4<9ZN4tj(@@bQOL0pgV;LrYxu;W2Q<~-8y0o>E zhiIYWRShTqf&7jp;@4RWt$LutZlw?^{Yin9gUK*v1%Pe6mdNm7DA8buS+6@cX*?k> zaT#K$Q_)40U1G{H_|zT?4H|}wrc$;Rpjk50$%3on)Axb#z~f#q^8mi!^t{nPP3w%h zRvdNWyzV*g+&?L^O3V$psHCVL3`uaDSs6Bo?NyoqjpDmS`|>DWxwt)Bop%Z{0nz+S zWSWb^W=2BcNBsH*Qgc8EZ0er!L0aL`hc=%DN(ez(@LAVWoFThZevKhzjrj}iQ!T~5aEzkVwRZjf; z_&Tj)D4#MJs>a%;8p4{~lVE2!sQDaHVV^HYpwit5LO?a%#=fBhlr`e&O9=!mxB&wLA}Fy zza*nvw*2_%OF>5FlrTX4{Z6!Tf43#%(n4@5n;R+c3cweT3})bCU*ikBsENL8=mbBX z9bK{kZ^Q1@_#-mzhO>}O{CUC?em2D%)G`G2^oY$;rI6Sq1-E)1z(74oJaw5}$(EN5 zP?Nk#9>YV*vYzi->QMVSfR#p0i6M11@c9RUvXg~P;x92gotZuTae29lDQeHQ;Iy=W#obMABxU6ABQ9X}6wIRRckG)~C5?LNyH3)5^A zT&WVuHifmenMoX|OJQ0$4|;1tzj-Fi!HH*f4JuB53iF1gURJvnSVE#?FxISRi-^sP zj#fo?_Gw~*taD5j^0Vs+&)I1cYhH_lUaqHJTJQd9Rc+&cJlIo~Aj%_EfeB_rtNK8JrHPAX?@3G+>ZiK3Ft zWO+LH1a|eZx@+r_V;-H!7Ncf$kEk$*l>65$dToZ_A9LR*3)_^`l`=ziisf7&zfDA| zvAynlpq654@F6|5i5f=f`@JoXTEv2*D4iCZcbF&PzC1!br@iNcRJ*=(gX^)1WO0I5 zAcfcdWBE}vP6%g0qpb%k{yTazjybE-gnkXVQCL*+?SgZ8#{Mx<))W+oIN0Wah+U=g^cX(;T0d$+R@s^9vztYp>XbJ%3MWf z0(*Qixj&0~y0TvVKisJ2#}g&FSoA?u z=kN|!#M`jmR;TaVlfQS$B46Q*Dmg7Or;T#PI`CUsFdEI9?lp%nWg5yy#jt-|%P=UY z*>+-k%deBIMzjntB$O$!X$LAvEk%FreP-r(F-AH3q0`mr`ik)3QVTHybE#TOo{@v%AtlYjFXdl0;id z61}%QR?vx*S+bCoGT**!TM_m3+THoUkc6Yyax_#JJ?fCvCEE3)c;bMlMUluaa`9QM zgY(70xeHs>`!D>Zv)@7b*0crR@*1^4!18X!nzLOPF-plXY#ag1F!< zBD=(w@jqmnrS&R$U>f|fkP5jx_xmHuC$+SP>#vf7XFquk#jZ9EQ7Y2hO(*VtMe*?r z@h2Ut?IHuzxO_ApKZEo$e1U%joN1@eMaPtXwiZ6wfeqWSbKoGvR6Qo=N9OG344;ZL zvFIoRb{_YBqV>z=CP~7L*q-h@=9TiFLm=^PB1blbvZgjphQi-f*vVCG!=VT!6Ia1s z$gabDKJcFViK#hFSNQYH1o|k~?YT9W+DuV;){;x48l;6j-*qIm_zS__Mj~U>?DU<2 z>cgHq%@s?Up-B(iRH1Dz={)ybls`v?(Tdh_*gyy6W4$86J0+89T8|E6&6=0))>m$k zal8gpJyoj9E;_ky%A{*Nb&<8REp)xJGj~%jU~kDM2Ehy(cl2Rt0(s$lN|5iE*qsKs zg;e7}!Hj%7_zMt$rjoJa3C0*VUy@WvwR2C`2}muF>+&%Jn?FT#Xf)!50am!sz8TI>|E=If$5V^_w(tF8| zTgjD8)&x#$n-8PJ>O}=784*pGf=eIP23^i6ipq`)&XP7&izj&n`57YDf6~qxi9p=% zXicjRGp+*kAA9T8ZRt?nF@#P3Vs=l+m$~b21Apo@ zo{zm&gp4~nIIk$UbwrZhLK7FY@F&iu`JkDcepB0bHb)X9BrARgImnb(5)ajKWpu|-zd}zi_MB7c2-B`p$DE{!**r;wDd)u!2rG;fF=~)lQHDCdJ+tmR zVaSb**^sf5e5mhBEanuR*m=jipW7hSInM|y_blm&xXWfhSoAGgFp=MdwTBn%3D{5lu_Lwi6fr*57dz)Y6P(AnUEV(>lM*4KH$%xf6unJ6{%P^!+stVn zU!kU%KiM9Lac3!^p_78v=*;0qc(~o4sl&(-sA)&*+cs+)r6dOn^dSvb>ldt%8yxi@Ucfv0d2A0 z*1ET;xoCEyl6zO91lUe(!b^!b>w8V$trE}bz-LPWoNqQ@E-`BdNBtPJY{V8cY4aDK z8U(*Oa{F$g6I5xW2rm3xgbtNnbm%+J$H4+qS6-+HZ5{2I&+apxCjJl$D`cB1v?Uze zJW8Ejf6juZy5-v8?2-J0NIk~kkXO!@eFlTl(K@O@c6UYu5BWTFZA$vkrDS_+M6KI4 z6RU>4Ib?IlYO>68;=V>#L z@$PcqsS;dV0gFlv-bpK*s#UYw;m-nRn!g;^M{Uz}KG+yWtZuy;zDggu6sA7tLzT$b z)6*PcpVs(QZ+?6IE;%yD?CoM(QWF8;*DB(RJ3BNzJye&bZz%Fmvc~HCW*dC^T0wjd zC6RK_Y3h&TAi^&12Z>2K+kWjcd;M6^EGt$K$T?x>*H~9z5&!Fix6gp>vTo=PRWw$0 zv32AX&Pu8@GKh3bAQl5v!3$pwe=l__THe2VeSRo&jv)`6nad+-bN>~;5q9ANinaZa z(gFy-5`<6vWJ8lP*X&^D>EuJ!?#&}GRIBkq=p9?@g_b|77;bPXex(ztmB<{A|4X+2 zs>G#{^@FR9J9$0*pnZOD70WT~06T|=i5Ag~^3RxI9^Hel+d2qOr+@{Cnvo9WqvuMh zdheKMu{H?)t8O$EK}akF4P<&%k>%4y0<)H3Vvu9yz}Il z!kk_y3Ek4N0zL6K3RGisft6a*6%&(u#^$Ty)#JLIRyZZ~p;r71RB*tDB_uX?&A8 z%A{GJ#qXd;eMO8GQV)vpZ=ARt%J_0}2INZrBNFl!n^gxkG=GdNhS|Ws=Pv&=>Xb~3 z5INILdT>YMoZ=(D+`?4TEqR5uXMU;~4s#P;y9D3-ef3fqc+1`JH{nESJ2sVSPIdXa zv_{ppd7~Yg;DJo%vSN+tLwDd(9R%@DeYAZ!U-;O=9MFo7&kQQ;;{2-;&DYru-9Jsq zADHqpoLaxR{|l5511ZclZ9ZI7yQ2p)i5n~)HG~>@#4=?jC?M(5q}-?==u_oO;fN+% zP^2>JgVL=aQU3=voF^R~+pM}4+|C+gX%2V(k&nNQCSoBB6jcd2i;l;+*tO8pPsJ(R zCDYy!$Nzpc@%V=Tj_=G`_~mzfxKx$vV}%z3{TTSQP)|IClR63zVD(W199S6_D}6-= z?Z&%twBY0U7imu8-b|5ktai?^$9hf?o2;cMNpn?RtTWM`JO*FD>@&D(Izp)g z2{;D{(Lu8%Jt*inuON4W4LC*Wt+@HP|8RwPx@E|%)$)xcIkkWDmWhYQ2N?Jxv3_MT zbVS5rt;uVKIcUBwM$!)k2)4iu3{v!mOzyD?9?VFNYGrV&BBVYx(PcD2NHx#|U7gVFLJl zvbjaxuA*P^WdZSR`?#g9qtu;mQiK{I6V$5x>)9Lpfr+T_i!Pu^7*&+xCtk7n7QZBh zg-O!$wsyG}eR1NqHmXa08dbWU_r)meaO+opkuB1Hku48z=nwy6wo3*oMUmqVwFqhu z$_`KZ5tl;JW#_vLrqS1K2*nSoQE-29&Pm6t)h~3s8kViqmKf7Vb0PGP0iy^hem?+7 zp)SuYW2WQX362DOOnL19mgVaie^RLn*KL+~eW1xG`-r4A>RhaG6*~NOBy0W}-hxwc z4fp`x{p-y7$KsgnF`xtU<*&^M;>X#6n@1T=0p5;MsY0FQ91Aw^jkihNM3Np2#M*3W zcDGQw6}5B0yKUFakB|D7Xuu1@#$)0agI=i{u&(E}%pnX@Rgy!cF?ZUb<$z^K4a34%od%KI($dR#kk`s}v;LfaTvksXF?Vsg*JEPOzC7$Ek zZzi}nJu_!J;2~xLa zAms6%LVW!)01t~H@ELXlv0&Rb7}L|2HN=3Mh*6cCW#6+p^PODz3DU@^3+(U(_{G@| z4hu1n6M(s?yrUQLgC3?5=L*#M53bEkjF5J|ecE{Z*i^>#z3du2&O-;3@0i0|a;Y7T zo;bl5Vejd_s1h%atK6JLugo8}1*V|4ZHAr7FP@?7FTp^gB+gJ$c6V;LPc?wJMEU-h zuNRRx{U5LJch?TIVATcByUx5blbp6FNP2sBr0|q8@%6(BK6xk!?2AI5{4bYWh9r2T3GY+n8J&8;* z3<>L=L`ugkrO=*hi{J3_S~#`hxen=ZQ0uid=Qt_+oS`r2gd3H5>&A#BsY39A=WAyc z8a*GX457feDFO9#m&YE?Fu#%^_P^+nxkzjgAi4RLoQ z3~|(T{1e@Y_U0mu&m5B4%7;9b41&Ne?s?Dn%N83!s2<*25_}JHIfvtdg_q`XLfpFX z5cro;#UU{pL$&)ZC|eTg*6yrZ!s?$~SJ1G+qQ{6DPz zME*eq)ODO@GJb)*de$I~o{jYn?%WCJCRVqGinh6}bjcA<065^;`I zh@EnlTbg9h%q&u+oHRf4tX};YfXXi>Mr&VOs_=N80-{;q0m+XBT-`B91R>T;;7kWj z6s9aT2y8)7Ts>Il67IR=#pQWf7R_eKPrB9xRqmiuuXZwdM7QOg9K+Zz6KfjX{FKR=G23WLx~SYC1)OLaHCeOu6`Fc zBet|4)I>Mx!5 0: - # Send each record to OpenAI for schema extraction - openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/tesrt/completions?api-version=2023-09-15-preview' - openai_headers = { - 'Content-Type': 'application/json', - 'api-key': 'ebe64320148849aead404cc3aec9cc49' - } - - for record in json_data: - openai_payload = { - "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {record}", - "max_tokens": 800, - "temperature": 0.2, - "frequency_penalty": 0, - "presence_penalty": 0, - "top_p": 1, - "stop": None - } - - try: - openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() - - # Extract only the text part from the OpenAI response - response_text = openai_response.json()['choices'][0]['text'] - print(response_text) - except requests.exceptions.HTTPError as errh: - print(f"HTTP Error: {errh}") - except requests.exceptions.RequestException as err: - print(f"Request Exception: {err}") - else: - print("No records found in the JSON data.") -else: - print(f"API call to fetch data failed with status code {response.status_code}") diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1ebba1d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -openai==0.28.0 -fuzzywuzzy==0.18.0 -fastapi==0.100.1 -pandas==2.0.3 -motor==3.2.0 -pydantic==2.1.1 -uvicorn==0.23.2 -annotated-types==0.5.0 -anyio==3.7.1 -click==8.1.6 -colorama==0.4.6 -dnspython==2.4.1 -h11==0.14.0 -idna==3.4 -pydantic_core==2.4.0 -pymongo==4.4.1 -python-dateutil==2.8.2 -pytz==2023.3 -setuptools==65.5.0 -six==1.16.0 -sniffio==1.3.0 -starlette==0.27.0 -typing_extensions==4.7.1 -tzdata==2023.3 -python-multipart -PyYAML==6.0 diff --git a/s.html b/s.html deleted file mode 100644 index a1ff5a9..0000000 --- a/s.html +++ /dev/null @@ -1,188 +0,0 @@ - - - - - Chat with Bot - - - -

Chat with Bot

-
- - - - - - - - - - -
- -
- - - - diff --git a/script.js b/script.js deleted file mode 100644 index 475768b..0000000 --- a/script.js +++ /dev/null @@ -1,30 +0,0 @@ -const list1 = ["FirstName", "Display Name", "Email Address", "Id", "address"]; -const list2 = ["fullname", "firstname", "company mail", "employeeId", "office address"]; - -const ul1 = document.getElementById("list1"); -const ul2 = document.getElementById("list2"); -const lines = document.getElementById("lines"); - -list1.forEach((item, index) => { - const li = document.createElement("li"); - li.textContent = item; - ul1.appendChild(li); - - const matchingIndex = list2.indexOf(item); - if (matchingIndex !== -1) { - const line = document.createElementNS("http://www.w3.org/2000/svg", "line"); - line.setAttribute("x1", "0"); - line.setAttribute("y1", `${li.offsetTop + li.offsetHeight / 2}`); - line.setAttribute("x2", "100%"); - line.setAttribute("y2", `${ul2.children[matchingIndex].offsetTop + ul2.children[matchingIndex].offsetHeight / 2}`); - line.setAttribute("stroke", "#000"); - line.setAttribute("stroke-width", "2"); - lines.appendChild(line); - } -}); - -list2.forEach((item, index) => { - const li = document.createElement("li"); - li.textContent = item; - ul2.appendChild(li); -}); diff --git a/sp.html b/sp.html deleted file mode 100644 index 9bbc807..0000000 --- a/sp.html +++ /dev/null @@ -1,190 +0,0 @@ - - - - - Chat with Bot - - - -

Chat with Bot

-
- - - - - - - - - - -
- -
-
- - - - \ No newline at end of file diff --git a/ss.py b/ss.py deleted file mode 100644 index f2bb89e..0000000 --- a/ss.py +++ /dev/null @@ -1,42 +0,0 @@ -similar_elements = [{'element_name_l1': ' Id', 'element_name_l2': ' Id'}, {'element_name_l1': 'First Name', 'element_name_l2': 'Firstname'}, {'element_name_l1': 'Last Name', 'element_name_l2': 'Lastname'}, {'element_name_l1': 'Hod Email Id', 'element_name_l2': 'Email'}, {'element_name_l1': 'Job Role', 'element_name_l2': 'Mobile'}, {'element_name_l1': 'Full Name', 'element_name_l2': 'Lastname'}, {'element_name_l1': 'Marital Status', 'element_name_l2': 'Status'}, {'element_name_l1': 'Office Country', 'element_name_l2': 'Country'}, {'element_name_l1': 'Current Country', 'element_name_l2': 'Country'}, {'element_name_l1': 'Current State', 'element_name_l2': 'Created'}] - - -def parse_elements(elements_list): - l1_matched = [] - l2_matched = [] - - for element in elements_list: - l1_matched.append(element['element_name_l1']) - l2_matched.append(element['element_name_l2']) - - return l1_matched, l2_matched - -l1_matched, l2_matched = parse_elements(similar_elements) -print("L1 Matched:", l1_matched) -print() -print("L2 Matched:", l2_matched) - -def find_unmatched_elements(original_list, matched_list): - unmatched = [element for element in original_list if element not in matched_list] - return unmatched - -# Example usage: -l1_original = ['Id','First Name','Last Name','Band','Employee Id','Business Unit','Ctc','Cost Center Id','Date Of Exit','Date Of Joining','Date Of Activation','Departments Hierarchy','Direct Manager Employee Id','Employee Type','Grade','Group Company','Hod Email Id','Hod','Hod Employee Id','Hrbp Employee Id', 'Job Role', 'Job Role Code', 'Job Role Name', 'Job Level Code', 'L2 Manager Employee Id', 'Separation Agreed Last Date', 'Separation Status', 'Total Ctc', 'Candidate Id', 'Center Type', 'Company Email Id', 'Date Of Birth', 'Date Of Resignation', 'City Type', 'Dependents', 'Education Details', 'Emergency Contact Number', 'Emergency Contact Person', 'Emergency Contact Relation', 'Full Name', 'Gender', 'Location Type', 'Marital Status', 'Middle Name', 'Office Address', 'Office Area', 'Office City', 'Office Country', 'Office Location', 'Office State', 'Primary Mobile Number', 'User Unique Id', 'Work Area Code', 'Current Address', 'Current Country', 'Current State', 'Personal Email Id', 'Personal Mobile No', 'Office Mobile No', 'Bank Name', 'Bank Branch', 'Bank Account', 'Bank Ifsc', 'Bank Pan', 'Aadhaar Number', 'Job Level', 'Function', 'Leadership Group', 'Nationality', 'Function Code', 'Direct Manager Email', 'Direct Manager Name', 'Permanent Pin Code', 'Latest Modified Any Attribute', 'Regular/ Temporary', 'Blood Group', 'Departments Hierarchy With Codes', 'Notice Period Assigned', 'Salutation', 'Permanent Address', 'Permanent City', 'Permanent Country', 'Permanent State', 'Date Of Death', 'Anniversary Date', 'Group Company Code', 'Functional Head', 'Employment Type', 'Contract End Date', 'Contract Start Date', 'Attendance Shift', 'Role', 'Dependent Role', 'Cost Center', 'Office Location Cost Center', 'Is Rehire', 'Rehire By', 'Rehire On', 'Rehire Reason', 'Employee Separation Reason', 'Admin Deactivation Type', 'Dm Pool', 'Home', 'Sub Employee Type', 'Hrbp Email Id', 'Source', 'Source Type', 'Fnf Status', 'Past Work', 'Passport Address', 'Passport Country', 'Passport Number', 'Passport Type', 'Federation Id', "Father'S Name", 'Job Role Alias', 'Channel', 'Fls/Nfls', 'Short Desg.', 'Zone', 'Region', 'Recruiter', 'Zrm', 'Hrbp', 'Zhr', 'Uan Number', 'Esic Number', 'Pf Number', 'Activation Timestamp', 'Designation Title', 'Latest Modified Timestamp'] - -l2_original = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] - - -l1_matched = ['Id', 'First Name', 'Last Name', 'Hod Email Id', 'Job Role', 'Full Name', 'Marital Status', 'Office Country', 'Current Country', 'Current State'] -l2_matched = [' Id', 'Firstname', 'Lastname', 'Email', 'Mobile', 'Lastname', 'Status', 'Country', 'Country', 'Created'] - - -l1_unmatched = find_unmatched_elements(l1_original, [elem.strip() for elem in l1_matched]) -l2_unmatched = find_unmatched_elements(l2_original, [elem.strip() for elem in l2_matched]) - -print() -print("Unmatched Elements L1:", l1_unmatched) -print() -print("Unmatched Elements L2:", l2_unmatched) - - - diff --git a/style.css b/style.css deleted file mode 100644 index af10e32..0000000 --- a/style.css +++ /dev/null @@ -1,35 +0,0 @@ -body { - font-family: Arial, sans-serif; - } - - .container { - display: flex; - justify-content: space-between; - align-items: center; - height: 100vh; - padding: 20px; - } - - .list { - border: 1px solid #ccc; - border-radius: 5px; - padding: 10px; - flex: 1; - max-height: 80vh; - overflow-y: auto; - } - - .left { - margin-right: 10px; - } - - .right { - margin-left: 10px; - } - - svg { - position: absolute; - top: 0; - left: 0; - } - \ No newline at end of file diff --git a/templates/index.html b/templates/index.html deleted file mode 100644 index 20039dc..0000000 --- a/templates/index.html +++ /dev/null @@ -1,227 +0,0 @@ - - - - - Chat with Bot - - - -

Chat with Bot

-
- - - - - - - - - - -
- -
-
- - - - \ No newline at end of file diff --git a/templates/input.html b/templates/input.html deleted file mode 100644 index 1d0e55f..0000000 --- a/templates/input.html +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - Data Processing - - - -

Data Processing

-
- - - -
- - \ No newline at end of file diff --git a/templates/output.html b/templates/output.html deleted file mode 100644 index 7e4f831..0000000 --- a/templates/output.html +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - Data Processing Result - - - -

Data Processing Result

- - - - - - - - - - - {% for record in response_data %} - - - - - - - {% endfor %} - -
jsonPathlabeldataTypevalue
{{ record.jsonPath }}{{ record.label }}{{ record.dataType }}{{ record.value }}
- Go Back - - \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index 7a311fc..0000000 --- a/test.py +++ /dev/null @@ -1,79 +0,0 @@ -import requests -from pymongo import MongoClient -import json - -def fetch_sample_record(collection): - data = list(collection.find()) - if len(data) > 0: - return data[0] - return None - -# Connect to MongoDB for darvin_user.new_user collection -mongo_connection_string_darvin = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" -mongo_client_darvin = MongoClient(mongo_connection_string_darvin) -mongo_db_darvin = mongo_client_darvin['darvin_user'] -mongo_collection_darvin = mongo_db_darvin['new_user'] - -# Connect to MongoDB for bugfix-avi.user collection -# Connect to MongoDB for bugfix-avi.user collection -mongo_connection_string_bugfix = "mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority" -mongo_client_bugfix = MongoClient(mongo_connection_string_bugfix) -mongo_db_bugfix = mongo_client_bugfix['bugfix-avi'] -mongo_collection_bugfix = mongo_db_bugfix['user'] - -# Fetch JSON data from darvin_user.new_user collection -sample_record_darvin = fetch_sample_record(mongo_collection_darvin) - -if sample_record_darvin: - # Send the sample record to OpenAI for schema extraction - openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' - openai_headers = { - 'Content-Type': 'application/json', - 'api-key': 'ebe64320148849aead404cc3aec9cc49' - } - - openai_payload = { - "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record_darvin}", - "max_tokens": 800, - "temperature": 0.2, - "frequency_penalty": 0, - "presence_penalty": 0, - "top_p": 1, - "stop": None - } - - try: - openai_response = requests.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() - - # Extract only the text part from the OpenAI response - openai_output = openai_response.json()['choices'][0]['text'] - - # Extract labels from the OpenAI output - openai_labels = [label.strip() for label in openai_output.split('\n') if label] - - # Fetch JSON data from bugfix-avi.user collection - sample_record_bugfix = fetch_sample_record(mongo_collection_bugfix) - - if sample_record_bugfix: - # Convert the JSON string to a dictionary - sample_record_bugfix = json.loads(sample_record_bugfix) - - # Extract labels from the sample record in bugfix-avi.user - bugfix_labels = [entry['label'] for entry in sample_record_bugfix] - - # Compare labels - similar_labels = set(openai_labels).intersection(set(bugfix_labels)) - - # Print similar labels - print("Similar Labels:") - print(similar_labels) - else: - print("No records found in bugfix-avi.user collection.") - except requests.exceptions.HTTPError as errh: - print(f"HTTP Error: {errh}") - except requests.exceptions.RequestException as err: - print(f"Request Exception: {err}") - -else: - print("No records found in the MongoDB collection.") diff --git a/test1.py b/test1.py deleted file mode 100644 index be2f9b3..0000000 --- a/test1.py +++ /dev/null @@ -1,92 +0,0 @@ -from fastapi import FastAPI, Form, HTTPException -import openai -import json - -openai.api_type = "azure" -openai.api_base = "https://cymetriopen.openai.azure.com/" -openai.api_version = "2023-07-01-preview" -openai.api_key = "ebe64320148849aead404cc3aec9cc49" - -app = FastAPI() - -# Maintain conversation history -conversation_history = [] -stored_responses = [] - -def get_response_text(response): - try: - # Attempt to get the text content from the response - text_content = response['choices'][0]['message']['content'] - return text_content - except (KeyError, IndexError): - return None - -def format_mapped_elements(response): - mapped_elements = [] - try: - # Extracting the mapped elements from the string response - start_index = response.find("{") - end_index = response.rfind("}") - json_content = response[start_index:end_index + 1] - json_data = json.loads(json_content) - - for key, value in json_data.items(): - mapped_elements.append(f"{{\"l1 element\": \"{key}\", \"l2 element\": \"{value}\"}}") - - return mapped_elements - except (KeyError, ValueError): - return None - - -def chat_with_bot(l1, l2, syntax): - # Concatenate l1, l2, and syntax into a single input string - input_text = f"{l1}\n{l2}\n{syntax}" - - # Initialize the conversation with system messages - message_text = [ - {"role": "system", "content": "You are an AI assistant that helps people find information."}, - ] - - # Append user inputs to the conversation - message_text.append({"role": "user", "content": input_text}) - - # Call OpenAI to get a response - completion = openai.ChatCompletion.create( - engine="tesrt", - messages=message_text, - temperature=0.7, - max_tokens=800, - top_p=0.95, - frequency_penalty=0, - presence_penalty=0, - stop=None - ) - - # Get the assistant's response - response = get_response_text(completion) - print("Response received from OpenAI:", response) - - # Format the mapped elements from the response - formatted_elements = format_mapped_elements(response) - - conversation_history.append({"role": "assistant", "content": formatted_elements}) - stored_responses.append(formatted_elements) # Automatically store the response - return formatted_elements - -# FastAPI endpoint for chatbot -@app.post("/chat") -def chat_endpoint(l1: str = Form(...), l2: str = Form(...), syntax: str = Form(...)): - bot_reply = chat_with_bot(l1, l2, syntax) - return {"bot_reply": bot_reply} - -# FastAPI endpoint to get stored responses -@app.get("/get_responses") -def get_responses(): - if not stored_responses: - raise HTTPException(status_code=404, detail="No stored responses") - return {"stored_responses": stored_responses} - -# Run the FastAPI server -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file diff --git a/tt.txt b/tt.txt deleted file mode 100644 index a58c580..0000000 --- a/tt.txt +++ /dev/null @@ -1,62 +0,0 @@ -Completionnnnnnnnn: { - "id": "chatcmpl-8LpWjW1XIuZoZAe6hvMLje1V19cdJ", - "object": "chat.completion", - "created": 1700213649, - "model": "gpt-35-turbo", - "prompt_filter_results": [ - { - "prompt_index": 0, - "content_filter_results": { - "hate": { - "filtered": false, - "severity": "safe" - }, - "self_harm": { - "filtered": false, - "severity": "safe" - }, - "sexual": { - "filtered": false, - "severity": "safe" - }, - "violence": { - "filtered": false, - "severity": "safe" - } - } - } - ], - "choices": [ - { - "index": 0, - "finish_reason": "stop", - "message": { - "role": "assistant", - "content": "Hello! How can I assist you today?" - }, - "content_filter_results": { - "hate": { - "filtered": false, - "severity": "safe" - }, - "self_harm": { - "filtered": false, - "severity": "safe" - }, - "sexual": { - "filtered": false, - "severity": "safe" - }, - "violence": { - "filtered": false, - "severity": "safe" - } - } - } - ], - "usage": { - "prompt_tokens": 25, - "completion_tokens": 9, - "total_tokens": 34 - } -} \ No newline at end of file From 624cfece208401f7bce59be4753912de8909a963 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 26 Dec 2023 11:35:14 +0530 Subject: [PATCH 06/91] files updated --- fetch_labels.py | 278 +++++++++++++++++++++-------------------------- requirements.txt | 26 +++++ 2 files changed, 150 insertions(+), 154 deletions(-) create mode 100644 requirements.txt diff --git a/fetch_labels.py b/fetch_labels.py index 2fc27a6..d6cd3b6 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -1,27 +1,27 @@ import uvicorn from fastapi import FastAPI, Form, Request, HTTPException -from fastapi.templating import Jinja2Templates import httpx import json import logging from fastapi import FastAPI, Form -from fastapi.responses import JSONResponse, HTMLResponse +from fastapi.responses import JSONResponse from fuzzywuzzy import fuzz from typing import List, Union import uvicorn from typing import List, Dict, Union app = FastAPI() -templates = Jinja2Templates(directory="templates") #logging.basicConfig(level=logging.DEBUG) - + def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] def compare_lists_with_fuzzy(l1, l2, threshold=50): + + matching_elements_l1 = [] matching_elements_l2 = [] non_matching_elements_l1 = [] @@ -32,9 +32,12 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): matching_element_l2 = '' for element_l2 in l2: + el1 = str(element_l1).lower() + el2 = str(element_l2).lower() similarity = fuzz.ratio( - str(element_l1).lower(), str(element_l2).lower() - ) # Convert to lowercase for case-insensitive comparison + el1, el2 + ) + print(f'l1 {el1} ;;; l2 {el2} ;; similarity {similarity}') # Convert to lowercase for case-insensitive comparison if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 @@ -67,158 +70,116 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: final_response = [] processed_labels = set() - + # Create a dictionary for easy lookup of response_data based on labels response_lookup = {data['label']: data for data in response_data} - + for element in similar_elements: # Find matching element in response_data based on label matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) - + if matched_data: final_response.append({ 'jsonPath': matched_data['jsonPath'], 'l2_matched': element['element_name_l2'], - 'datatype': matched_data['dataType'], + 'datatype': matched_data['datatype'], 'value': matched_data['value'] }) processed_labels.add(element['element_name_l1']) # Track processed labels - + # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: final_response.append({ 'jsonPath': data['jsonPath'], 'l2_matched': '', # No match from l2 - 'datatype': data['dataType'], + 'datatype': data['datatype'], 'value': data['value'] # Use value from response_data }) - + return final_response - - + + + @app.post('/process_data') -async def process_data(request: Request, data_url: str = Form(...)): +async def process_data(request: Request, data: dict): try: - #logging.debug(f"Processing data from URL: {data_url}") + print("data :",data) + + def extract_info(json_data): + appId = json_data.get("appId") + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + url = json_data["schema"]["nodes"][0]["data"]["url"] + request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + + print("appId: ",appId) + print("body: ",body) + print("headers: ",headers) + #print("url: ",url) + print("request_method: ",request_method) + print("params: ",params) + return appId, body, headers, url, request_method, params + + appId, body, headers, url, request_method, params = extract_info(data) + + + final_url = url + "?domain=kathreftis.ai" + + print("final_url: ",final_url) # Fetch JSON data from the specified URL using httpx for asynchronous requests async with httpx.AsyncClient() as client: - ''' - - - body -: -"{\n \"size\":10,\n \"page\":1,\n \"filter\":\"*\"\n}" -headers -: -Array(2) -0 -: -{key: 'Authorization', value: 'Bearer afsafasf', included: true} -1 -: -{key: 'Content-Type', value: 'application/json', included: true} -length -: -2 -[[Prototype]] -: -Array(0) -id -: -1 -label -: -"Node 1" -params -: -[] -parentProperties -: -[] -requestMethod -: -"GET" -response -: -"" -selectedNodeObj -: -[[Prototype]] -: -Object -selectedNodes -: -Array(0) -length -: -0 -[[Prototype]] -: -Array(0) -url -: -"" -[[Prototype]] -: -Object - ''' - - ### From lokesh's request -- you will get 4 things - # applicationId - # body --> which you have to convert from JSON string to dictionary - # headers --> convert from [] to dictionary - # url --> use as-is (will be string) - # requestMethod --> will be a string; use with match as given below else use if-else ladder - - - # check if headers is empty - ### Parse body --- {\n \"size\":10,\n \"page\":1,\n \"filter\":\"*\"\n} and take it as dictionary - #### Get url directly from requests - ### Get requestMethod from requestMethod - - ### you will get this as an array; convert each element of array - - body_dict = {} - header_dict = {} - url_from_request = "" - requestMethod_from_request = "" - - - client.headers = header_dict - - - - ### into dict; such that key in the array element is your dict key and value is corresponding value + #url_from_request = data["url"] + #print("url_from_request: ",url) + requestMethod_from_request = request_method + #print("requestMethod_from_request: ",requestMethod_from_request) + match requestMethod_from_request: case "GET": # Call method for GET request - response = client.get(url=request.data.url) - case "POST": - # Call method for POST request - ## if body is empty, skip the data field - ## if body is not empty, use as below - response = client.post(url=request.data.url, data=body_dict) - case "PUT": - # Handle other request methods - response = client.put(url=request.data.url, data=body_dict) - - case "PATCH": - response = client.patch(url=request.data.url, data=body_dict) - - case "DELETE": - response = client.delete(url=request.data.url) - - #response = await client.get(data_url) + response = await client.get(url=final_url, headers=headers) + print("response: ",response) + + # case "POST": + # # Call method for POST request + # ## if body is empty, skip the data field + # ## if body is not empty, use as below + # response = client.post(url=url_from_request) + # print("response: ",response) + # case "PUT": + # # Handle other request methods + # response = client.put(url=request.data.url, data=body_dict) + + # case "PATCH": + # response = client.patch(url=request.data.url, data=body_dict) + + # case "DELETE": + # response = client.delete(url=request.data.url) + if response.status_code >= 200 or response.status_code <= 204: # Assuming the response contains JSON data, you can parse it json_data = response.json() + print("json_data: ",json_data) + ''' + users_array = json_data['users'][0] + users_json = json.dumps(users_array, indent=4) + print(users_json) + print("shreyas") + ''' + json_data = json_data['users'] + + #print("json_data: ",json_data) if isinstance(json_data, list) and len(json_data) > 0: - sample_record = json_data[0] - + sample_record = json.dumps(json_data[0]) + #sample_dict = {} + #sample_dict['primaryEmail'] = sample_record['primaryEmail'] + print("sample_record: ",sample_record) + # Send the sample record to OpenAI for schema extraction openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' openai_headers = { @@ -227,8 +188,8 @@ async def process_data(request: Request, data_url: str = Form(...)): } openai_payload = { - "prompt": f"Give me list of fields as jsonPath and labels and datatype and value in this json sample {sample_record}", - "max_tokens": 800, + "prompt": "Give me list of fields as jsonPath and labels and datatype and value in this json sample in json format only "+sample_record, + "max_tokens": 8000, "temperature": 0.2, "frequency_penalty": 0, "presence_penalty": 0, @@ -237,47 +198,59 @@ async def process_data(request: Request, data_url: str = Form(...)): } try: - async with httpx.AsyncClient() as openai_client: - openai_response = await openai_client.post(openai_url, headers=openai_headers, json=openai_payload) - openai_response.raise_for_status() + print("declaring async client") + async with httpx.AsyncClient(timeout=360) as openai_client: + print("start of openai request") + print(f'url is {openai_url}') + #print(f'headers are {openai_headers}') + #print(f'payload is {openai_payload}') + openai_response = '' + try: + openai_response = await openai_client.post(openai_url, headers=openai_headers, json=openai_payload,timeout=300) + except Exception as e: + print(f'openai response {openai_response}') + print(e) + finally: + print(f'openai response {openai_response}') + #openai_response.raise_for_status() + print(f'Respoinse freom openai is {openai_response}') # Extract only the text part from the OpenAI response response_text = openai_response.json()['choices'][0]['text'] - - # Parse the response_text into a list of dictionaries + print(f'response text is {response_text}') + + #Parse the response_text into a list of dictionaries response_data = json.loads(response_text) - - #logging.debug(f"Received response from OpenAI: {response_data}") - - + print(f'response data is {response_data}') + logging.debug(f"Received response from OpenAI: {response_data}") + + l1 = [item['label'] for item in response_data] - + if isinstance(l1, str): l1_list = convert_string_to_list(l1) else: l1_list = l1 - + l2 = [' Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] - + if isinstance(l2, str): l2_list = convert_string_to_list(l2) else: l2_list = l2 - - threshold = 65 - + + threshold = 55 + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - + final_response = generate_final_response(result['similar_elements'], response_data) - + #print(f'final response is {final_response}') return JSONResponse(content=final_response) - - except httpx.HTTPError as errh: - #logging.error(f"HTTP Error: {errh}") - raise HTTPException(status_code=500, detail=f"HTTP Error: {errh}") - except httpx.RequestError as err: - #logging.error(f"Request Exception: {err}") - raise HTTPException(status_code=500, detail=f"Request Exception: {err}") + + except Exception as errh: + print(f"HTTP Error {errh} is: {errh}") + #raise HTTPException(status_code=500, detail=f"HTTP Error: {errh}") + else: return "No records found in the JSON data." else: @@ -285,10 +258,7 @@ async def process_data(request: Request, data_url: str = Form(...)): except Exception as e: #logging.error(f"An unexpected error occurred: {e}") raise HTTPException(status_code=500, detail=str(e)) - - - - - + + if __name__ == "__main__": uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1ebba1d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +openai==0.28.0 +fuzzywuzzy==0.18.0 +fastapi==0.100.1 +pandas==2.0.3 +motor==3.2.0 +pydantic==2.1.1 +uvicorn==0.23.2 +annotated-types==0.5.0 +anyio==3.7.1 +click==8.1.6 +colorama==0.4.6 +dnspython==2.4.1 +h11==0.14.0 +idna==3.4 +pydantic_core==2.4.0 +pymongo==4.4.1 +python-dateutil==2.8.2 +pytz==2023.3 +setuptools==65.5.0 +six==1.16.0 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.7.1 +tzdata==2023.3 +python-multipart +PyYAML==6.0 From ccb269f248c70227d905282704dc621fb2413d87 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 12:26:31 +0530 Subject: [PATCH 07/91] files updated for multi-tenancy --- config/loader.py | 2 +- database/connection.py | 2 +- fetch_labels.py | 36 ++++++++++++++++++++++++++++-------- requirements.txt | 1 + 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/config/loader.py b/config/loader.py index 33d83fe..2e17162 100644 --- a/config/loader.py +++ b/config/loader.py @@ -13,7 +13,7 @@ def getConfiguration(self): return self.data if platform.system() == 'Windows': - c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-rolemining/config.yaml") + c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") else: c = Configuration("config.yaml") diff --git a/database/connection.py b/database/connection.py index abce394..2072735 100644 --- a/database/connection.py +++ b/database/connection.py @@ -5,7 +5,7 @@ def load_configuration(): if platform.system() == 'Windows': - c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-genrativeAI/config.yaml") + c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") else: c = Configuration("config.yaml") return c.getConfiguration() diff --git a/fetch_labels.py b/fetch_labels.py index d6cd3b6..f82fd26 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -9,10 +9,19 @@ from typing import List, Union import uvicorn from typing import List, Dict, Union +from database.connection import get_collection app = FastAPI() -#logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) + +def stored_input(tenant: str): + #logging.debug(f"Getting collection for tenant: {tenant}") + return get_collection(tenant, "openai_input") + +def stored_response(tenant: str): + #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") + return get_collection(tenant, "openai_output") def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' @@ -54,11 +63,6 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): if element_l2.strip("'") not in matching_elements_l2 ] - # print("Matching Elements in l1:", matching_elements_l1) - # print("Matching Elements in l2:", matching_elements_l2) - # print("Non-Matching Elements in l1:", non_matching_elements_l1) - # print("Non-Matching Elements in l2:", non_matching_elements_l2) - similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) @@ -104,6 +108,12 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat @app.post('/process_data') async def process_data(request: Request, data: dict): try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + input_collection.insert_one(data) + logging.debug("Input respone saved successfully") print("data :",data) def extract_info(json_data): @@ -125,11 +135,15 @@ def extract_info(json_data): appId, body, headers, url, request_method, params = extract_info(data) + + final_url = url - final_url = url + "?domain=kathreftis.ai" + if params: + final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) print("final_url: ",final_url) + # Fetch JSON data from the specified URL using httpx for asynchronous requests async with httpx.AsyncClient() as client: #url_from_request = data["url"] @@ -155,7 +169,7 @@ def extract_info(json_data): # case "PATCH": # response = client.patch(url=request.data.url, data=body_dict) - + # case "DELETE": # response = client.delete(url=request.data.url) @@ -244,6 +258,12 @@ def extract_info(json_data): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) final_response = generate_final_response(result['similar_elements'], response_data) + final_response_dict = {"final_response": final_response} + + output_collection.insert_one(final_response_dict) + logging.debug("Final response saved successfully") + + #print(f'final response is {final_response}') return JSONResponse(content=final_response) diff --git a/requirements.txt b/requirements.txt index 1ebba1d..61dfd3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ typing_extensions==4.7.1 tzdata==2023.3 python-multipart PyYAML==6.0 +httpx \ No newline at end of file From b48aab91ca571feba4ec31bed5eec38f2f46c44c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 12:41:21 +0530 Subject: [PATCH 08/91] reading the yaml file from environment variable --- config/loader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config/loader.py b/config/loader.py index 2e17162..ea99ee5 100644 --- a/config/loader.py +++ b/config/loader.py @@ -1,5 +1,6 @@ import yaml import platform +import os class Configuration: def __init__(self, filepath): @@ -12,12 +13,15 @@ def __init__(self, filepath): def getConfiguration(self): return self.data -if platform.system() == 'Windows': +if platform.system() != 'Windows': c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") else: - c = Configuration("config.yaml") + #c = Configuration("config.yaml") + c = Configuration(os.environ.get("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml", "config.yaml")) ConfObject = c.getConfiguration() + + def getConfigObject(): return ConfObject \ No newline at end of file From 4951da52d6cff5a184ea8bb0f34ca9e66c9d5deb Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 12:53:12 +0530 Subject: [PATCH 09/91] now reading configueration file path from environment variable - CONFIG_FILE_PATH --- Dockerfile | 2 +- config/loader.py | 43 +++++++++++++++++++++++++++--------------- database/connection.py | 5 +---- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 085223e..5fecd99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN pip install flask EXPOSE 5000 # Define environment variable -ENV NAME World +ENV CONFIG_FILE_PATH=/app/config.yaml # Run your FastAPI application CMD ["uvicorn", "fetch_labels:app", "--host", "0.0.0.0", "--port", "5000"] diff --git a/config/loader.py b/config/loader.py index ea99ee5..7605eb7 100644 --- a/config/loader.py +++ b/config/loader.py @@ -1,27 +1,40 @@ import yaml import platform +import os + +# class Configuration: +# def __init__(self, filepath): +# self.data = {} +# with open(filepath, "r") as yamlfile: +# data_loaded = yaml.safe_load(yamlfile) + +# self.data = data_loaded + +# def getConfiguration(self): +# return self.data + +# if platform.system() != 'Windows': +# c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") +# else: +# #c = Configuration("config.yaml") +# c = Configuration(os.environ.get("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml", "config.yaml")) + +# ConfObject = c.getConfiguration() + + + + import os class Configuration: - def __init__(self, filepath): + def __init__(self, filepath=None): self.data = {} + if filepath is None: + filepath = os.environ.get('CONFIG_FILE_PATH', '/app/config.yaml') with open(filepath, "r") as yamlfile: data_loaded = yaml.safe_load(yamlfile) - self.data = data_loaded def getConfiguration(self): return self.data - -if platform.system() != 'Windows': - c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") -else: - #c = Configuration("config.yaml") - c = Configuration(os.environ.get("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml", "config.yaml")) - -ConfObject = c.getConfiguration() - - - -def getConfigObject(): - return ConfObject \ No newline at end of file + \ No newline at end of file diff --git a/database/connection.py b/database/connection.py index 2072735..ce5cbda 100644 --- a/database/connection.py +++ b/database/connection.py @@ -4,10 +4,7 @@ import platform def load_configuration(): - if platform.system() == 'Windows': - c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") - else: - c = Configuration("config.yaml") + c = Configuration() return c.getConfiguration() data = load_configuration() From 86a941e02efa5b4ddc32e15bd5ff6eba7b42b1eb Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 12:56:25 +0530 Subject: [PATCH 10/91] added prefix to APIs generativeaisrvc --- fetch_labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index f82fd26..8febedf 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -105,7 +105,7 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat -@app.post('/process_data') +@app.post('/generativeaisrvc/process_data') async def process_data(request: Request, data: dict): try: tenant = "generativeAI" From 387214d3e20349e40adef44677f9b49ec0414e8a Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 12:58:34 +0530 Subject: [PATCH 11/91] changed order of Dockerfile --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5fecd99..bef14f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ FROM python:3.10-slim # Set the working directory to /app WORKDIR /app -# Copy the contents of the local directory into the container at /app -COPY . /app +COPY requirements.txt /app/requirements.txt # Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt @@ -12,11 +11,17 @@ RUN pip install --no-cache-dir -r requirements.txt # Install Flask RUN pip install flask + # Make port 5000 available to the world outside this container EXPOSE 5000 # Define environment variable ENV CONFIG_FILE_PATH=/app/config.yaml + + +# Copy the contents of the local directory into the container at /app +COPY . /app + # Run your FastAPI application CMD ["uvicorn", "fetch_labels:app", "--host", "0.0.0.0", "--port", "5000"] From c9283e51406dfd22b228a5d302b38ac5d2b06c14 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 13:16:29 +0530 Subject: [PATCH 12/91] handled missing body, params, headers --- fetch_labels.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/fetch_labels.py b/fetch_labels.py index 8febedf..8519be8 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -118,13 +118,28 @@ async def process_data(request: Request, data: dict): def extract_info(json_data): appId = json_data.get("appId") - body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) - headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + body = "" + params = None + headers = None + try: + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + except: + print("body not found") + + + try: + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + except: + print("headers not found") + url = json_data["schema"]["nodes"][0]["data"]["url"] request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] - params_list = json_data["schema"]["nodes"][0]["data"]["params"] - params = {param["key"]: param["value"] for param in params_list if param["included"]} - + try: + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + except: + print("params not found") + print("appId: ",appId) print("body: ",body) print("headers: ",headers) From a574da71bdf3aec870d3615ba59aaa3dead97c77 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Dec 2023 13:23:56 +0530 Subject: [PATCH 13/91] added levenshtein distance to see if that helps the performance, also removed some similarity logs --- fetch_labels.py | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fetch_labels.py b/fetch_labels.py index 8519be8..b3a645f 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -46,7 +46,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similarity = fuzz.ratio( el1, el2 ) - print(f'l1 {el1} ;;; l2 {el2} ;; similarity {similarity}') # Convert to lowercase for case-insensitive comparison + #print(f'l1 {el1} ;;; l2 {el2} ;; similarity {similarity}') # Convert to lowercase for case-insensitive comparison if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 diff --git a/requirements.txt b/requirements.txt index 61dfd3b..01ca7b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,5 @@ typing_extensions==4.7.1 tzdata==2023.3 python-multipart PyYAML==6.0 -httpx \ No newline at end of file +httpx +python-Levenshtein \ No newline at end of file From 85d305331754b17679a2c43ca1e816a9e4bba888 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 13:37:59 +0530 Subject: [PATCH 14/91] changed insert to upsert for input collections --- fetch_labels.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index b3a645f..f8393d2 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -112,7 +112,13 @@ async def process_data(request: Request, data: dict): input_collection = stored_input(tenant) output_collection = stored_response(tenant) - input_collection.insert_one(data) + #input_collection.insert_one(data) + input_collection.update_one( + {"appId": data["appId"]}, + {"$set": data}, + upsert=True + ) + logging.debug("Input respone saved successfully") print("data :",data) From fc98cc86bba9e0ee081cf43b35ddaf9963e6406f Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 13:40:15 +0530 Subject: [PATCH 15/91] added new api just to gget sample --- fetch_labels.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index f8393d2..8d2601b 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -103,7 +103,124 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat return final_response + +@app.post('/generativeaisrvc/send_and_get_sample') +async def process_data(request: Request, data: dict): + response_data = {} + try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + #input_collection.insert_one(data) + input_collection.update_one( + {"appId": data["appId"]}, + {"$set": data}, + upsert=True + ) + + logging.debug("Input respone saved successfully") + print("data :",data) + + def extract_info(json_data): + appId = json_data.get("appId") + body = "" + params = None + headers = None + try: + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + except: + print("body not found") + + + try: + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + except: + print("headers not found") + + url = json_data["schema"]["nodes"][0]["data"]["url"] + request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] + try: + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + except: + print("params not found") + + print("appId: ",appId) + print("body: ",body) + print("headers: ",headers) + #print("url: ",url) + print("request_method: ",request_method) + print("params: ",params) + return appId, body, headers, url, request_method, params + + appId, body, headers, url, request_method, params = extract_info(data) + + + final_url = url + + if params: + final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) + + print("final_url: ",final_url) + + # Fetch JSON data from the specified URL using httpx for asynchronous requests + async with httpx.AsyncClient() as client: + #url_from_request = data["url"] + #print("url_from_request: ",url) + requestMethod_from_request = request_method + #print("requestMethod_from_request: ",requestMethod_from_request) + + match requestMethod_from_request: + case "GET": + # Call method for GET request + response = await client.get(url=final_url, headers=headers) + print("response: ",response) + + # case "POST": + # # Call method for POST request + # ## if body is empty, skip the data field + # ## if body is not empty, use as below + # response = client.post(url=url_from_request) + # print("response: ",response) + # case "PUT": + # # Handle other request methods + # response = client.put(url=request.data.url, data=body_dict) + + # case "PATCH": + # response = client.patch(url=request.data.url, data=body_dict) + + # case "DELETE": + # response = client.delete(url=request.data.url) + + + if response.status_code >= 200 or response.status_code <= 204: + # Assuming the response contains JSON data, you can parse it + json_data = response.json() + print("json_data: ",json_data) + ''' + users_array = json_data['users'][0] + users_json = json.dumps(users_array, indent=4) + print(users_json) + print("shreyas") + ''' + json_data = json_data['users'] + + #print("json_data: ",json_data) + + if isinstance(json_data, list) and len(json_data) > 0: + sample_record = json.dumps(json_data[0]) + #sample_dict = {} + #sample_dict['primaryEmail'] = sample_record['primaryEmail'] + print("sample_record: ",sample_record) + response_data = sample_record + + except Exception as e: + print(e) + + return response_data + @app.post('/generativeaisrvc/process_data') async def process_data(request: Request, data: dict): @@ -118,7 +235,7 @@ async def process_data(request: Request, data: dict): {"$set": data}, upsert=True ) - + logging.debug("Input respone saved successfully") print("data :",data) From 1f61bda6b6e0b4bd168858fc6b6649c4682008d6 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 13:42:15 +0530 Subject: [PATCH 16/91] sending data as json --- fetch_labels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index 8d2601b..ded3a4f 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -219,7 +219,8 @@ def extract_info(json_data): except Exception as e: print(e) - return response_data + #return response_data + return JSONResponse(content=response_data, media_type="application/json") @app.post('/generativeaisrvc/process_data') From f740205b98d1bfef8891a0eba62c26afe88bf7ba Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 13:44:33 +0530 Subject: [PATCH 17/91] sample data response cleanup --- fetch_labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index ded3a4f..5d8bcc5 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -214,7 +214,7 @@ def extract_info(json_data): #sample_dict = {} #sample_dict['primaryEmail'] = sample_record['primaryEmail'] print("sample_record: ",sample_record) - response_data = sample_record + response_data = json_data[0] except Exception as e: print(e) From 53783f915ebcce5cd96544fbb5f31bd79b60ab30 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 26 Dec 2023 16:54:14 +0530 Subject: [PATCH 18/91] prompt change for datatype field in response. --- fetch_labels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fetch_labels.py b/fetch_labels.py index 5d8bcc5..c2e9f88 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -341,7 +341,7 @@ def extract_info(json_data): } openai_payload = { - "prompt": "Give me list of fields as jsonPath and labels and datatype and value in this json sample in json format only "+sample_record, + "prompt": "Give me list of fields as jsonPath and labels and datatype and value in this json sample in json format only,keep all fields in lowercase only"+sample_record, "max_tokens": 8000, "temperature": 0.2, "frequency_penalty": 0, @@ -420,4 +420,4 @@ def extract_info(json_data): if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file + uvicorn.run(app, host="127.0.0.1", port=5001) \ No newline at end of file From 91606ba5d04b4f4fb0fc2748cf376cc7f450d96f Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 17:06:33 +0530 Subject: [PATCH 19/91] removed space in Id in l2 and also changed final response from insert to update --- fetch_labels.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index c2e9f88..d770399 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -385,7 +385,7 @@ def extract_info(json_data): else: l1_list = l1 - l2 = [' Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] if isinstance(l2, str): l2_list = convert_string_to_list(l2) @@ -400,6 +400,14 @@ def extract_info(json_data): final_response_dict = {"final_response": final_response} output_collection.insert_one(final_response_dict) + final_response_dict['appId'] = appId + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") From 3821d87e727757e8c2dce897742690dfdc112ab9 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Tue, 26 Dec 2023 17:13:25 +0530 Subject: [PATCH 20/91] fixed the code where both insert and upsert were being done --- fetch_labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_labels.py b/fetch_labels.py index d770399..61d0329 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -399,7 +399,7 @@ def extract_info(json_data): final_response = generate_final_response(result['similar_elements'], response_data) final_response_dict = {"final_response": final_response} - output_collection.insert_one(final_response_dict) + #output_collection.insert_one(final_response_dict) final_response_dict['appId'] = appId output_collection.update_one( {"appId": appId}, From 469ac506465e6af61a1257c122ae83cc107cda24 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 8 Mar 2024 15:22:42 +0530 Subject: [PATCH 21/91] files added for policy mapping --- config/loader.py | 2 +- datatype_extract_logic.py | 101 ++++++++++++ dev.py | 332 ++++++++++++++++++++++++++++++++++++++ fetch_labels.py | 4 +- new.py | 79 +++++++++ policy_mapping.py | 332 ++++++++++++++++++++++++++++++++++++++ sample.py | 225 ++++++++++++++++++++++++++ users_extract_logic.py | 69 ++++++++ 8 files changed, 1141 insertions(+), 3 deletions(-) create mode 100644 datatype_extract_logic.py create mode 100644 dev.py create mode 100644 new.py create mode 100644 policy_mapping.py create mode 100644 sample.py create mode 100644 users_extract_logic.py diff --git a/config/loader.py b/config/loader.py index 7605eb7..c5a6bac 100644 --- a/config/loader.py +++ b/config/loader.py @@ -30,7 +30,7 @@ class Configuration: def __init__(self, filepath=None): self.data = {} if filepath is None: - filepath = os.environ.get('CONFIG_FILE_PATH', '/app/config.yaml') + filepath = os.environ.get('CONFIG_FILE_PATH', 'config.yaml') with open(filepath, "r") as yamlfile: data_loaded = yaml.safe_load(yamlfile) self.data = data_loaded diff --git a/datatype_extract_logic.py b/datatype_extract_logic.py new file mode 100644 index 0000000..2c88dfb --- /dev/null +++ b/datatype_extract_logic.py @@ -0,0 +1,101 @@ +from datetime import datetime +from dateutil.parser import parse +import json + +def get_distinct_keys_and_datatypes(json_data): + distinct_keys_datatypes = {} + + def explore_json(obj): + if isinstance(obj, dict): + for key, value in obj.items(): + if key not in distinct_keys_datatypes: + distinct_keys_datatypes[key] = get_data_type(value) + elif distinct_keys_datatypes[key] != get_data_type(value): + distinct_keys_datatypes[key] = 'Mixed' + explore_json(value) + elif isinstance(obj, list): + for item in obj: + explore_json(item) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if parse_result.hour == 0 and parse_result.minute == 0 and parse_result.second == 0: + return 'date' # Date if no time components + else: + return 'datetime' # Datetime if time components present + except (ValueError, OverflowError): + return 'string' # Fallback to string if parsing as date/datetime fails + else: + return type(value).__name__ + + explore_json(json_data) + return distinct_keys_datatypes + + +# Test JSON data +json_data = { + "users": [ + { + "id": 2, + "name": "Jane Smith", + "age": 25, + "email": "jane@example.com", + "is_active": True, + "profile_picture": "https://example.com/profile.jpg", + "registration_date": "2022-01-15", + "last_login": "2024-03-04T15:30:00", + "address": { + "street": "456 Elm St", + "city": "Othertown", + "state": "NY", + "zipcode": "54321" + }, + "interests": ["swimming", "cooking"], + "ldap_datetime_custom": "20240305110000Z", + "ldap_byte_array": "0x0A0B0C", + "password": "securepassword", + "successfactor_date": "2024-03-04T12:45:00+00:00", + "custom": "Another custom data" + }, + { + "PersonId": 300000003107430, + "PersonNumber": "0010188", + "CorrespondenceLanguage": "null", + "BloodType": "B+", + "DateOfBirth": "1990-11-01", + "DateOfDeath": "null", + "CountryOfBirth": "null", + "RegionOfBirth": "null", + "TownOfBirth": "null", + "ApplicantNumber": "null", + "CreatedBy": "FUSION_APPS_HCM_ESS_LOADER_APPID", + "CreationDate": "2020-01-10T11:02:11+00:00", + "LastUpdatedBy": "Ali.Mehaboob", + "LastUpdateDate": "2021-11-10T17:18:46.144+00:00", + }, + { + "businessPhones": [], + "displayName": "cyrus", + "givenName": "cyrus", + "jobTitle": "Unemployed", + "mail": "null", + "mobilePhone": "7977280292", + "officeLocation": "null", + "preferredLanguage": "null", + "surname": "gracias", + "userPrincipalName": "cyrus@cymmetri.com", + "id": "353db878-b8b6-4235-adfa-f8979d7eeeb2" + }, + ] +} + + +parsed_json = json.loads(json.dumps(json_data)) +distinct_keys_datatypes = get_distinct_keys_and_datatypes(parsed_json) +l1 = list(distinct_keys_datatypes.keys()) +print("all the keys",l1) +for key, datatype in distinct_keys_datatypes.items(): + print(f"Key: {key}, Data Type: {datatype}") \ No newline at end of file diff --git a/dev.py b/dev.py new file mode 100644 index 0000000..de49ea2 --- /dev/null +++ b/dev.py @@ -0,0 +1,332 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection +from dateutil.parser import parse +from datetime import datetime +from datetime import date +import datetime +import json + +app = FastAPI() + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + +def stored_input(tenant: str): + #logging.debug(f"Getting collection for tenant: {tenant}") + return get_collection(tenant, "input") + +def stored_response(tenant: str): + #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") + return get_collection(tenant, "output") + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +#-----------------------------extracting the user object from response----------------- + +def extract_user_data(response): + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data_list + +#---------------------------extracting keys, datatype, label and jsonpath---------------- + +def get_distinct_keys_and_datatypes(json_data): + logging.debug(f"extracting the properties from the json data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) + else: + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' + + + explore_json(json_data) + return distinct_keys_datatypes + +#-------------------fuzzy logic matching function---------------------- + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + logging.debug(f"comparing logic for list1 and list2") + + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + el1 = str(element_l1).lower() + el2 = str(element_l2).lower() + similarity = fuzz.ratio( + el1, el2 + ) + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + +#----------------------generates final response--------------- +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: + logging.debug(f"Beautifying the response for saving into the collection") + final_response = [] + processed_labels = set() + + # Create a dictionary for easy lookup of response_data based on labels + response_lookup = {data['label']: data for data in response_data} + + for element in similar_elements: + # Find matching element in response_data based on label + matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) + + if matched_data: + final_response.append({ + 'jsonPath': matched_data['jsonpath'], + 'l2_matched': element['element_name_l2'], + 'datatype': matched_data['datatype'], + 'value': matched_data['value'] + }) + processed_labels.add(element['element_name_l1']) # Track processed labels + + else: + print(f"No matched data found for {element['element_name_l1']}") + + # Handle unmatched elements from l1 + for data in response_data: + if data['label'] not in processed_labels: + final_response.append({ + 'jsonPath': data['jsonpath'], + 'l2_matched': '', # No match from l2 + 'datatype': data['datatype'], + 'value': data['value'] # Use value from response_data + }) + + return final_response + + +#----------------------api for policy mapping----------------------------- +@app.post('/generativeaisrvc/get_policy_mapped') +async def get_mapped(request: Request, data: dict): + logging.debug(f"API call for auto policy mapping with the application") + try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + #input_collection.insert_one(data) + input_collection.update_one( + {"appId": data["appId"]}, + {"$set": data}, + upsert=True + ) + + logging.debug("Input respone saved successfully") + print("data :",data) + + def extract_info(json_data): + appId = json_data.get("appId") + body = "" + params = None + headers = None + try: + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + except: + print("body not found") + try: + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + except: + print("headers not found") + + url = json_data["schema"]["nodes"][0]["data"]["url"] + request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] + try: + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + except: + print("params not found") + + print("appId: ",appId) + print("body: ",body) + print("headers: ",headers) + #print("url: ",url) + print("request_method: ",request_method) + print("params: ",params) + return appId, body, headers, url, request_method, params + + appId, body, headers, url, request_method, params = extract_info(data) + + + final_url = url + + if params: + final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) + + print("final_url: ",final_url) + + + # Fetch JSON data from the specified URL using httpx for asynchronous requests + async with httpx.AsyncClient() as client: + requestMethod_from_request = request_method + + match requestMethod_from_request: + case "GET": + # Call method for GET request + response = await client.get(url=final_url, headers=headers) + + if response.status_code >= 200 or response.status_code <= 204: + # Assuming the response contains JSON data, you can parse it + json_data = response.json() + json_data_ = extract_user_data(json_data) + print("json_data: ",json_data_) + + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + + l1 = [item['label'] for item in response_data] + + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) + + l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + threshold = 60 + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + final_response = generate_final_response(result['similar_elements'], response_data) + final_response_dict = {"final_response": final_response} + + final_response_dict['appId'] = appId + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + return JSONResponse(content=final_response) + + else: + raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/fetch_labels.py b/fetch_labels.py index 61d0329..9de400a 100644 --- a/fetch_labels.py +++ b/fetch_labels.py @@ -323,7 +323,7 @@ def extract_info(json_data): print(users_json) print("shreyas") ''' - json_data = json_data['users'] + #json_data = json_data['users'] #print("json_data: ",json_data) @@ -428,4 +428,4 @@ def extract_info(json_data): if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=5001) \ No newline at end of file + uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/new.py b/new.py new file mode 100644 index 0000000..128d0fa --- /dev/null +++ b/new.py @@ -0,0 +1,79 @@ +import csv +from pymongo import MongoClient +import json +from openai import OpenAI +import logging + +client = OpenAI(api_key='ebe64320148849aead404cc3aec9cc49') + + +# Set up MongoDB connection +mongo_client = MongoClient("mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority") +db = mongo_client["cymmetri-development"] +collection = db["audit_log"] + +logging.basicConfig(level=logging.DEBUG) + + +def extract_entities_from_description(description: str): + return {"entities": [(0, len(description), "ENTITY")]} + +def generate_training_data_with_prompt(description): + openai_url = 'https://cymetriopen.openai.azure.com/openai/deployments/instructionalmodel/completions?api-version=2023-09-15-preview' + openai_headers = { + 'Content-Type': 'application/json', + 'api-key': 'ebe64320148849aead404cc3aec9cc49' + } + + openai_payload = { + "prompt": f"Extract entities like USER_ID, ACTION, RESULT from the data: {description}", + "max_tokens": 8000, + "temperature": 0.2, + "frequency_penalty": 0, + "presence_penalty": 0, + "top_p": 1, + "stop": None + } + + openai_response = client.completions.create(engine="text-davinci-003", + prompt=json.dumps(openai_payload), + headers=openai_headers) + + entities_response = openai_response.choices[0].text.strip() + entities = extract_entities_from_description(entities_response) + return entities + +def process_data_in_batches_and_generate_training_set(batch_size=100): + try: + total_documents = collection.count_documents({}) + num_batches = (total_documents + batch_size - 1) // batch_size + + training_data = [] + + for batch_num in range(num_batches): + batch = list(collection.find().skip(batch_num * batch_size).limit(batch_size)) + for item in batch: + description = item.get("description", "") + entities = generate_training_data_with_prompt(description) + training_data.append((description, entities)) + + # Save each batch separately if needed + save_to_csv(training_data, f"batch_{batch_num + 1}.csv") + training_data = [] + + # Save all data to a single CSV file + save_to_csv(training_data, "all_data.csv") + + print("Processing completed successfully") + except Exception as e: + print(f"Error: {e}") + +def save_to_csv(training_data, csv_file): + with open(csv_file, 'a', newline='', encoding='utf-8') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(["text", "entities"]) + for text, entities in training_data: + csvwriter.writerow([text, json.dumps(entities)]) + +if __name__ == "__main__": + process_data_in_batches_and_generate_training_set() diff --git a/policy_mapping.py b/policy_mapping.py new file mode 100644 index 0000000..de49ea2 --- /dev/null +++ b/policy_mapping.py @@ -0,0 +1,332 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection +from dateutil.parser import parse +from datetime import datetime +from datetime import date +import datetime +import json + +app = FastAPI() + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + +def stored_input(tenant: str): + #logging.debug(f"Getting collection for tenant: {tenant}") + return get_collection(tenant, "input") + +def stored_response(tenant: str): + #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") + return get_collection(tenant, "output") + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +#-----------------------------extracting the user object from response----------------- + +def extract_user_data(response): + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data_list + +#---------------------------extracting keys, datatype, label and jsonpath---------------- + +def get_distinct_keys_and_datatypes(json_data): + logging.debug(f"extracting the properties from the json data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) + else: + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' + + + explore_json(json_data) + return distinct_keys_datatypes + +#-------------------fuzzy logic matching function---------------------- + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + logging.debug(f"comparing logic for list1 and list2") + + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + el1 = str(element_l1).lower() + el2 = str(element_l2).lower() + similarity = fuzz.ratio( + el1, el2 + ) + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + +#----------------------generates final response--------------- +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: + logging.debug(f"Beautifying the response for saving into the collection") + final_response = [] + processed_labels = set() + + # Create a dictionary for easy lookup of response_data based on labels + response_lookup = {data['label']: data for data in response_data} + + for element in similar_elements: + # Find matching element in response_data based on label + matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) + + if matched_data: + final_response.append({ + 'jsonPath': matched_data['jsonpath'], + 'l2_matched': element['element_name_l2'], + 'datatype': matched_data['datatype'], + 'value': matched_data['value'] + }) + processed_labels.add(element['element_name_l1']) # Track processed labels + + else: + print(f"No matched data found for {element['element_name_l1']}") + + # Handle unmatched elements from l1 + for data in response_data: + if data['label'] not in processed_labels: + final_response.append({ + 'jsonPath': data['jsonpath'], + 'l2_matched': '', # No match from l2 + 'datatype': data['datatype'], + 'value': data['value'] # Use value from response_data + }) + + return final_response + + +#----------------------api for policy mapping----------------------------- +@app.post('/generativeaisrvc/get_policy_mapped') +async def get_mapped(request: Request, data: dict): + logging.debug(f"API call for auto policy mapping with the application") + try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + #input_collection.insert_one(data) + input_collection.update_one( + {"appId": data["appId"]}, + {"$set": data}, + upsert=True + ) + + logging.debug("Input respone saved successfully") + print("data :",data) + + def extract_info(json_data): + appId = json_data.get("appId") + body = "" + params = None + headers = None + try: + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + except: + print("body not found") + try: + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + except: + print("headers not found") + + url = json_data["schema"]["nodes"][0]["data"]["url"] + request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] + try: + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + except: + print("params not found") + + print("appId: ",appId) + print("body: ",body) + print("headers: ",headers) + #print("url: ",url) + print("request_method: ",request_method) + print("params: ",params) + return appId, body, headers, url, request_method, params + + appId, body, headers, url, request_method, params = extract_info(data) + + + final_url = url + + if params: + final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) + + print("final_url: ",final_url) + + + # Fetch JSON data from the specified URL using httpx for asynchronous requests + async with httpx.AsyncClient() as client: + requestMethod_from_request = request_method + + match requestMethod_from_request: + case "GET": + # Call method for GET request + response = await client.get(url=final_url, headers=headers) + + if response.status_code >= 200 or response.status_code <= 204: + # Assuming the response contains JSON data, you can parse it + json_data = response.json() + json_data_ = extract_user_data(json_data) + print("json_data: ",json_data_) + + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + + l1 = [item['label'] for item in response_data] + + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) + + l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + threshold = 60 + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + final_response = generate_final_response(result['similar_elements'], response_data) + final_response_dict = {"final_response": final_response} + + final_response_dict['appId'] = appId + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + return JSONResponse(content=final_response) + + else: + raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/sample.py b/sample.py new file mode 100644 index 0000000..326c97d --- /dev/null +++ b/sample.py @@ -0,0 +1,225 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection +from datetime import datetime +from dateutil.parser import parse +import json +from dateutil.parser import parse +import datetime +from datetime import date +from dateutil.parser import ParserError + +app = FastAPI() + + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + +def stored_input(tenant: str): + #logging.debug(f"Getting collection for tenant: {tenant}") + return get_collection(tenant, "input") + +def stored_response(tenant: str): + #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") + return get_collection(tenant, "output") + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +def get_distinct_keys_and_datatypes(json_data): + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) + else: + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' + + + explore_json(json_data) + return distinct_keys_datatypes + +#-----------------------------extracting the user object from response----------------- + +def extract_user_data(response): + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data_list + +#----------------------api for policy mapping----------------------------- +@app.post('/generativeaisrvc/get_policy_mapped') +async def get_mapped(request: Request, data: dict): + try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + #input_collection.insert_one(data) + input_collection.update_one( + {"appId": data["appId"]}, + {"$set": data}, + upsert=True + ) + + logging.debug("Input respone saved successfully") + print("data :",data) + + def extract_info(json_data): + appId = json_data.get("appId") + body = "" + params = None + headers = None + try: + body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) + except: + print("body not found") + + + try: + headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} + except: + print("headers not found") + + url = json_data["schema"]["nodes"][0]["data"]["url"] + request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] + try: + params_list = json_data["schema"]["nodes"][0]["data"]["params"] + params = {param["key"]: param["value"] for param in params_list if param["included"]} + except: + print("params not found") + + print("appId: ",appId) + print("body: ",body) + print("headers: ",headers) + #print("url: ",url) + print("request_method: ",request_method) + print("params: ",params) + return appId, body, headers, url, request_method, params + + appId, body, headers, url, request_method, params = extract_info(data) + + + final_url = url + + if params: + final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) + + print("final_url: ",final_url) + + + # Fetch JSON data from the specified URL using httpx for asynchronous requests + async with httpx.AsyncClient() as client: + #url_from_request = data["url"] + #print("url_from_request: ",url) + requestMethod_from_request = request_method + #print("requestMethod_from_request: ",requestMethod_from_request) + + match requestMethod_from_request: + case "GET": + # Call method for GET request + response = await client.get(url=final_url, headers=headers) + #print(f"type of response {type(response)}") + #print("response: ",response) + + if response.status_code >= 200 or response.status_code <= 204: + # Assuming the response contains JSON data, you can parse it + json_data = response.json() + json_data_ = extract_user_data(json_data) + #print("json_data: ",json_data_) + logging.debug(f"type of json_data is {type(json_data)}") + + response_data = get_distinct_keys_and_datatypes(json_data_) + #return list(response_data.values()) + return response_data + + else: + raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file diff --git a/users_extract_logic.py b/users_extract_logic.py new file mode 100644 index 0000000..3f505a7 --- /dev/null +++ b/users_extract_logic.py @@ -0,0 +1,69 @@ +import json + +def extract_user_data(response): + user_data = {} + + def is_user_data(obj): + # Check if object contains common user data keys + user_keys = {'displayName', 'givenName', 'mail', 'id'} + return all(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data + if isinstance(obj, dict): + if is_user_data(obj): + user_data = obj + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data + + +# Example response data +response = { + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users", + "value": [ + { + "businessPhones": [], + "displayName": "Aakanksha Raina", + "givenName": "Aakanksha", + "jobTitle": "Null", + "mail": "aakanksha.raina@cymmetri.com", + "mobilePhone": "null", + "officeLocation": "null", + "preferredLanguage": "null", + "surname": "Raina", + "userPrincipalName": "aakanksha.raina@cymmetri.com", + "id": "c835ea75-707b-4e63-b4dc-b24b4b522a09" + }, + { + "businessPhones": [], + "displayName": "Abhishek Ghante", + "givenName": "Abhishek", + "jobTitle": "null", + "mail": "abhishek.ghante@cymmetri.com", + "mobilePhone": "null", + "officeLocation": "null", + "preferredLanguage": "null", + "surname": "Ghante", + "userPrincipalName": "abhishek.ghante@cymmetri.com", + "id": "c39d5cf7-2d6c-4281-b58b-451125761c3d" + } + ] +} + +#json_data = response.json() + +# Parse the user data from the response +user_data = extract_user_data(response) + +# Print the extracted user data +print(type(user_data)) +print(user_data) From 86816e1d472f3395b5b1fba724e218649e6c7455 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 8 Mar 2024 17:49:43 +0530 Subject: [PATCH 22/91] updated the docker file with working file name as policy_mapping:app --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index bef14f8..41000ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,17 +11,14 @@ RUN pip install --no-cache-dir -r requirements.txt # Install Flask RUN pip install flask - # Make port 5000 available to the world outside this container EXPOSE 5000 # Define environment variable ENV CONFIG_FILE_PATH=/app/config.yaml - - # Copy the contents of the local directory into the container at /app COPY . /app # Run your FastAPI application -CMD ["uvicorn", "fetch_labels:app", "--host", "0.0.0.0", "--port", "5000"] +CMD ["uvicorn", "policy_mapping:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file From d37525834ec550a315242229b94ff5480e8dab8a Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 8 Mar 2024 19:14:59 +0530 Subject: [PATCH 23/91] updated file with removed request and accepting as response. --- dev.py | 128 ++++++++++++++---------------------------------- new_approach.py | 44 +++++++++++++++++ 2 files changed, 81 insertions(+), 91 deletions(-) create mode 100644 new_approach.py diff --git a/dev.py b/dev.py index de49ea2..eb1bef3 100644 --- a/dev.py +++ b/dev.py @@ -212,118 +212,64 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(request: Request, data: dict): +async def get_mapped(data: dict): logging.debug(f"API call for auto policy mapping with the application") try: tenant = "generativeAI" input_collection = stored_input(tenant) output_collection = stored_response(tenant) - #input_collection.insert_one(data) - input_collection.update_one( - {"appId": data["appId"]}, - {"$set": data}, - upsert=True - ) + # Store the received response directly into the input collection + input_collection.insert_one(data) logging.debug("Input respone saved successfully") print("data :",data) - def extract_info(json_data): - appId = json_data.get("appId") - body = "" - params = None - headers = None - try: - body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) - except: - print("body not found") - try: - headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} - except: - print("headers not found") + + # Assuming the response contains JSON data, you can parse it + json_data = data + json_data_ = extract_user_data(json_data) + print("json_data: ",json_data_) - url = json_data["schema"]["nodes"][0]["data"]["url"] - request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] - try: - params_list = json_data["schema"]["nodes"][0]["data"]["params"] - params = {param["key"]: param["value"] for param in params_list if param["included"]} - except: - print("params not found") - - print("appId: ",appId) - print("body: ",body) - print("headers: ",headers) - #print("url: ",url) - print("request_method: ",request_method) - print("params: ",params) - return appId, body, headers, url, request_method, params - - appId, body, headers, url, request_method, params = extract_info(data) - + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) - final_url = url - - if params: - final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) - - print("final_url: ",final_url) - - - # Fetch JSON data from the specified URL using httpx for asynchronous requests - async with httpx.AsyncClient() as client: - requestMethod_from_request = request_method - - match requestMethod_from_request: - case "GET": - # Call method for GET request - response = await client.get(url=final_url, headers=headers) - - if response.status_code >= 200 or response.status_code <= 204: - # Assuming the response contains JSON data, you can parse it - json_data = response.json() - json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) - - response_data = get_distinct_keys_and_datatypes(json_data_) - #response_data=list(response_data.values()) - - l1 = [item['label'] for item in response_data] - - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) - else: - l1_list = set(l1) - print("list1: ",l1_list) + l1 = [item['label'] for item in response_data] - l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 + l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 - threshold = 60 + threshold = 60 - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - final_response = generate_final_response(result['similar_elements'], response_data) - final_response_dict = {"final_response": final_response} + final_response = generate_final_response(result['similar_elements'], response_data) + final_response_dict = {"final_response": final_response} - final_response_dict['appId'] = appId - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) + # Assuming 'appId' is present in the received response + appId = data.get("appId") + final_response_dict['appId'] = appId - logging.debug("Final response saved successfully") + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) - return JSONResponse(content=final_response) + logging.debug("Final response saved successfully") - else: - raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + return JSONResponse(content=final_response) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/new_approach.py b/new_approach.py new file mode 100644 index 0000000..790627a --- /dev/null +++ b/new_approach.py @@ -0,0 +1,44 @@ +@app.post('/generativeaisrvc/get_policy_mapped') +async def get_mapped(data: dict): + logging.debug(f"API call for auto policy mapping with the application") + try: + tenant = "generativeAI" + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + + # Store the received response directly into the input collection + input_collection.insert_one(data) + + logging.debug("Input response saved successfully") + + # Extract necessary information directly from the received response + json_data_ = extract_user_data(data) + response_data = get_distinct_keys_and_datatypes(json_data_) + + l1 = [item['label'] for item in response_data] + + # Predefined list l2 remains unchanged + + threshold = 60 + result = compare_lists_with_fuzzy(l1, l2, threshold) + + final_response = generate_final_response(result['similar_elements'], response_data) + final_response_dict = {"final_response": final_response} + + # Assuming 'appId' is present in the received response + appId = data.get("appId") + final_response_dict['appId'] = appId + + # Store the final response into the output collection + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + return JSONResponse(content=final_response) + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) From e936fe867e13427c48ad769e7951a3a4614fc37f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 8 Mar 2024 21:46:22 +0530 Subject: [PATCH 24/91] updated docker file --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 41000ad..95c00e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,4 +21,4 @@ ENV CONFIG_FILE_PATH=/app/config.yaml COPY . /app # Run your FastAPI application -CMD ["uvicorn", "policy_mapping:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file +CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file From 19e252c0e21feb6a3fcfca1248af88a25cb231a9 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Sat, 9 Mar 2024 10:05:11 +0530 Subject: [PATCH 25/91] host changed from 127.0.0.1. to 0.0.0.0 --- dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev.py b/dev.py index eb1bef3..193ac83 100644 --- a/dev.py +++ b/dev.py @@ -275,4 +275,4 @@ async def get_mapped(data: dict): raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000) From 554bbd8470f2580d3ec6c3793e9552d3532aef4d Mon Sep 17 00:00:00 2001 From: Shreyas Date: Sat, 9 Mar 2024 10:29:26 +0530 Subject: [PATCH 26/91] port updated to 8000 --- Dockerfile | 4 ++-- dev.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95c00e6..5ce60cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN pip install --no-cache-dir -r requirements.txt RUN pip install flask # Make port 5000 available to the world outside this container -EXPOSE 5000 +EXPOSE 8000 # Define environment variable ENV CONFIG_FILE_PATH=/app/config.yaml @@ -21,4 +21,4 @@ ENV CONFIG_FILE_PATH=/app/config.yaml COPY . /app # Run your FastAPI application -CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file +CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/dev.py b/dev.py index 193ac83..61f87ec 100644 --- a/dev.py +++ b/dev.py @@ -275,4 +275,4 @@ async def get_mapped(data: dict): raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) + uvicorn.run(app, host="0.0.0.0", port=8000) From 8c4f0763944b983df4e3531f35ec1887fe02184e Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 11 Mar 2024 10:41:26 +0530 Subject: [PATCH 27/91] port changed --- Dockerfile | 4 ++-- dev.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ce60cb..3144271 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ EXPOSE 8000 ENV CONFIG_FILE_PATH=/app/config.yaml # Copy the contents of the local directory into the container at /app -COPY . /app +COPY . /app # Run your FastAPI application -CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file diff --git a/dev.py b/dev.py index 61f87ec..193ac83 100644 --- a/dev.py +++ b/dev.py @@ -275,4 +275,4 @@ async def get_mapped(data: dict): raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000) + uvicorn.run(app, host="0.0.0.0", port=5000) From 1ffe9243c2e50f8557cd9f4aaa17453e4c1d3915 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 11 Mar 2024 13:00:22 +0530 Subject: [PATCH 28/91] files updated with logic of adding datatypes for l2 and added routes.yaml --- dev.py | 50 +++++++- new_approach.py | 292 ++++++++++++++++++++++++++++++++++++++++++++-- policy_mapping.py | 149 ++++++++++------------- routes.yaml | 32 +++++ 4 files changed, 419 insertions(+), 104 deletions(-) create mode 100644 routes.yaml diff --git a/dev.py b/dev.py index 193ac83..b20bc1b 100644 --- a/dev.py +++ b/dev.py @@ -173,7 +173,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): return result #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str]) -> List[Dict[str, Union[str, int]]]: logging.debug(f"Beautifying the response for saving into the collection") final_response = [] processed_labels = set() @@ -186,10 +186,12 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) if matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) final_response.append({ 'jsonPath': matched_data['jsonpath'], + 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], - 'datatype': matched_data['datatype'], + 'l2_datatype': l2_datatype, 'value': matched_data['value'] }) processed_labels.add(element['element_name_l1']) # Track processed labels @@ -202,8 +204,9 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat if data['label'] not in processed_labels: final_response.append({ 'jsonPath': data['jsonpath'], + 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 - 'datatype': data['datatype'], + 'l2_datatype': '', 'value': data['value'] # Use value from response_data }) @@ -243,8 +246,43 @@ async def get_mapped(data: dict): l1_list = set(l1) print("list1: ",l1_list) - l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] - + #l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'department', 'designation', 'appUpdatedDate' 'country', 'city' 'mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2 = ['Id','department', 'employeeId', 'appUpdatedDate', 'displayname', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfdBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2_datatypes = { + 'Id': 'INTEGER', + 'department': 'STRING', + 'employeeId': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayname': 'STRING', + 'firstName': 'STRING', + 'lastName': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'mobile': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'login': 'INTEGER', + 'userType': 'STRING', + 'dateOfdBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'status': 'STRING', + 'landline': 'STRING', + 'appUserId': 'STRING', + 'Created': 'DATETIME', + 'Updated': 'DATETIME', + 'Created By': 'STRING', + 'Updated By': 'STRING', + 'Assignedgroups': 'ARRAY', + 'Provisionedapps': 'ARRAY', + 'Attributes': 'CUSTOM', + 'Rbacroles': 'ARRAY', + 'Version': 'STRING', + 'Class': 'STRING' + } + if isinstance(l2, str): l2_list = convert_string_to_list(l2) else: @@ -254,7 +292,7 @@ async def get_mapped(data: dict): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - final_response = generate_final_response(result['similar_elements'], response_data) + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response diff --git a/new_approach.py b/new_approach.py index 790627a..b20bc1b 100644 --- a/new_approach.py +++ b/new_approach.py @@ -1,35 +1,304 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection +from dateutil.parser import parse +from datetime import datetime +from datetime import date +import datetime +import json + +app = FastAPI() + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + +def stored_input(tenant: str): + #logging.debug(f"Getting collection for tenant: {tenant}") + return get_collection(tenant, "input") + +def stored_response(tenant: str): + #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") + return get_collection(tenant, "output") + +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + + +#-----------------------------extracting the user object from response----------------- + +def extract_user_data(response): + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data_list + +#---------------------------extracting keys, datatype, label and jsonpath---------------- + +def get_distinct_keys_and_datatypes(json_data): + logging.debug(f"extracting the properties from the json data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) + else: + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' + + + explore_json(json_data) + return distinct_keys_datatypes + +#-------------------fuzzy logic matching function---------------------- + +def compare_lists_with_fuzzy(l1, l2, threshold=50): + logging.debug(f"comparing logic for list1 and list2") + + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + + for element_l2 in l2: + el1 = str(element_l1).lower() + el2 = str(element_l2).lower() + similarity = fuzz.ratio( + el1, el2 + ) + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + + result = {"similar_elements": similar_elements} + return result + +#----------------------generates final response--------------- +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str]) -> List[Dict[str, Union[str, int]]]: + logging.debug(f"Beautifying the response for saving into the collection") + final_response = [] + processed_labels = set() + + # Create a dictionary for easy lookup of response_data based on labels + response_lookup = {data['label']: data for data in response_data} + + for element in similar_elements: + # Find matching element in response_data based on label + matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) + + if matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + final_response.append({ + 'jsonPath': matched_data['jsonpath'], + 'l1_datatype': matched_data['datatype'], + 'l2_matched': element['element_name_l2'], + 'l2_datatype': l2_datatype, + 'value': matched_data['value'] + }) + processed_labels.add(element['element_name_l1']) # Track processed labels + + else: + print(f"No matched data found for {element['element_name_l1']}") + + # Handle unmatched elements from l1 + for data in response_data: + if data['label'] not in processed_labels: + final_response.append({ + 'jsonPath': data['jsonpath'], + 'l1_datatype': data['datatype'], + 'l2_matched': '', # No match from l2 + 'l2_datatype': '', + 'value': data['value'] # Use value from response_data + }) + + return final_response + + +#----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict): logging.debug(f"API call for auto policy mapping with the application") try: tenant = "generativeAI" - input_collection = stored_input(tenant) - output_collection = stored_response(tenant) + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) # Store the received response directly into the input collection input_collection.insert_one(data) - logging.debug("Input response saved successfully") + logging.debug("Input respone saved successfully") + print("data :",data) - # Extract necessary information directly from the received response - json_data_ = extract_user_data(data) + + # Assuming the response contains JSON data, you can parse it + json_data = data + json_data_ = extract_user_data(json_data) + print("json_data: ",json_data_) + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) l1 = [item['label'] for item in response_data] - - # Predefined list l2 remains unchanged + + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) + + #l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'department', 'designation', 'appUpdatedDate' 'country', 'city' 'mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2 = ['Id','department', 'employeeId', 'appUpdatedDate', 'displayname', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfdBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2_datatypes = { + 'Id': 'INTEGER', + 'department': 'STRING', + 'employeeId': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayname': 'STRING', + 'firstName': 'STRING', + 'lastName': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'mobile': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'login': 'INTEGER', + 'userType': 'STRING', + 'dateOfdBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'status': 'STRING', + 'landline': 'STRING', + 'appUserId': 'STRING', + 'Created': 'DATETIME', + 'Updated': 'DATETIME', + 'Created By': 'STRING', + 'Updated By': 'STRING', + 'Assignedgroups': 'ARRAY', + 'Provisionedapps': 'ARRAY', + 'Attributes': 'CUSTOM', + 'Rbacroles': 'ARRAY', + 'Version': 'STRING', + 'Class': 'STRING' + } + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + threshold = 60 - result = compare_lists_with_fuzzy(l1, l2, threshold) - final_response = generate_final_response(result['similar_elements'], response_data) + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response appId = data.get("appId") final_response_dict['appId'] = appId - # Store the final response into the output collection output_collection.update_one( {"appId": appId}, {"$set": final_response_dict}, @@ -42,3 +311,6 @@ async def get_mapped(data: dict): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) diff --git a/policy_mapping.py b/policy_mapping.py index de49ea2..91d86c4 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -173,7 +173,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): return result #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]]) -> List[Dict[str, Union[str, int]]]: +def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str]) -> List[Dict[str, Union[str, int]]]: logging.debug(f"Beautifying the response for saving into the collection") final_response = [] processed_labels = set() @@ -186,10 +186,12 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) if matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) final_response.append({ 'jsonPath': matched_data['jsonpath'], + 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], - 'datatype': matched_data['datatype'], + 'l2_datatype': l2_datatype, 'value': matched_data['value'] }) processed_labels.add(element['element_name_l1']) # Track processed labels @@ -202,8 +204,9 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat if data['label'] not in processed_labels: final_response.append({ 'jsonPath': data['jsonpath'], + 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 - 'datatype': data['datatype'], + 'l2_datatype': '', 'value': data['value'] # Use value from response_data }) @@ -229,101 +232,71 @@ async def get_mapped(request: Request, data: dict): logging.debug("Input respone saved successfully") print("data :",data) - def extract_info(json_data): - appId = json_data.get("appId") - body = "" - params = None - headers = None - try: - body = json.loads(json_data["schema"]["nodes"][0]["data"]["body"]) - except: - print("body not found") - try: - headers = {header["key"]: header["value"] for header in json_data["schema"]["nodes"][0]["data"]["headers"]} - except: - print("headers not found") - - url = json_data["schema"]["nodes"][0]["data"]["url"] - request_method = json_data["schema"]["nodes"][0]["data"]["requestMethod"] - try: - params_list = json_data["schema"]["nodes"][0]["data"]["params"] - params = {param["key"]: param["value"] for param in params_list if param["included"]} - except: - print("params not found") - - print("appId: ",appId) - print("body: ",body) - print("headers: ",headers) - #print("url: ",url) - print("request_method: ",request_method) - print("params: ",params) - return appId, body, headers, url, request_method, params - - appId, body, headers, url, request_method, params = extract_info(data) - + # Assuming the response contains JSON data, you can parse it + json_data = data + json_data_ = extract_user_data(json_data) + print("json_data: ",json_data_) - final_url = url - - if params: - final_url += "?" + "&".join(f"{key}={value}" for key,value in params.items()) - - print("final_url: ",final_url) - - - # Fetch JSON data from the specified URL using httpx for asynchronous requests - async with httpx.AsyncClient() as client: - requestMethod_from_request = request_method - - match requestMethod_from_request: - case "GET": - # Call method for GET request - response = await client.get(url=final_url, headers=headers) - - if response.status_code >= 200 or response.status_code <= 204: - # Assuming the response contains JSON data, you can parse it - json_data = response.json() - json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) - - response_data = get_distinct_keys_and_datatypes(json_data_) - #response_data=list(response_data.values()) - - l1 = [item['label'] for item in response_data] - - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) - else: - l1_list = set(l1) - print("list1: ",l1_list) + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) - l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + l1 = [item['label'] for item in response_data] - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) + + l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2_datatypes = { + 'Id': 'INTEGER', + 'Displayname': 'STRING', + 'Firstname': 'STRING', + 'Lastname': 'STRING', + 'Country': 'STRING', + 'Mobile': 'STRING', + 'Email': 'STRING', + 'Status': 'STRING', + 'Created': 'DATETIME', + 'Updated': 'DATETIME', + 'Created By': 'STRING', + 'Updated By': 'STRING', + 'Assignedgroups': 'ARRAY', + 'Provisionedapps': 'ARRAY', + 'Attributes': 'CUSTOM', + 'Rbacroles': 'ARRAY', + 'Version': 'STRING', + 'Class': 'STRING' + } + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 - threshold = 60 + threshold = 60 - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - final_response = generate_final_response(result['similar_elements'], response_data) - final_response_dict = {"final_response": final_response} + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) + final_response_dict = {"final_response": final_response} - final_response_dict['appId'] = appId - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) + # Assuming 'appId' is present in the received response + #appId = data.get("appId") + #final_response_dict['appId'] = appId - logging.debug("Final response saved successfully") + output_collection.update_one( + #{"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) - return JSONResponse(content=final_response) + logging.debug("Final response saved successfully") - else: - raise HTTPException(status_code=response.status_code, detail=f"API call to fetch data failed with status code {response.status_code}") + return JSONResponse(content=final_response) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/routes.yaml b/routes.yaml new file mode 100644 index 0000000..21c47e4 --- /dev/null +++ b/routes.yaml @@ -0,0 +1,32 @@ +--- +api: /v1/config/projects/{project}/routing/ingress/{id} +meta: + id: generativeaiservice + project: unotech +spec: + modify: + headers: [] + outputFormat: yaml + requestTemplate: "" + responseTemplate: "" + template: go + project: unotech + rule: + rule: allow + source: + hosts: + - '*' + methods: + - '*' + port: 0 + rewrite: /generativeaisrvc/get_policy_mapped + type: prefix + url: /generativeaisrvc/get_policy_mapped + targets: + - host: generativeaiservice.unotech.svc.cluster.local + port: 8080 + scheme: http + type: "" + version: "" + weight: 100 +type: ingress-route From 639a34a963043f27d010db888ab0839c7ce97dea Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 11 Mar 2024 14:54:25 +0530 Subject: [PATCH 29/91] attributeName field added to show the labels of list1 --- dev.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/dev.py b/dev.py index b20bc1b..e101a81 100644 --- a/dev.py +++ b/dev.py @@ -189,6 +189,7 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat l2_datatype = l2_datatypes.get(element['element_name_l2'], None) final_response.append({ 'jsonPath': matched_data['jsonpath'], + 'attributeName': element['element_name_l1'], 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], 'l2_datatype': l2_datatype, @@ -204,6 +205,7 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat if data['label'] not in processed_labels: final_response.append({ 'jsonPath': data['jsonpath'], + 'attributeName': data['label'], 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 'l2_datatype': '', @@ -246,41 +248,36 @@ async def get_mapped(data: dict): l1_list = set(l1) print("list1: ",l1_list) - #l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'department', 'designation', 'appUpdatedDate' 'country', 'city' 'mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] - l2 = ['Id','department', 'employeeId', 'appUpdatedDate', 'displayname', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfdBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { 'Id': 'INTEGER', 'department': 'STRING', 'employeeId': 'STRING', + 'designation': 'STRING', 'appUpdatedDate': 'DATETIME', - 'displayname': 'STRING', - 'firstName': 'STRING', - 'lastName': 'STRING', + 'displayName': 'STRING', + 'mobile': 'STRING', 'country': 'STRING', 'city': 'STRING', - 'mobile': 'STRING', 'email': 'STRING', 'end_date': 'DATE', + 'firstName': 'STRING', + 'login': 'INTEGER', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', 'login': 'INTEGER', 'userType': 'STRING', - 'dateOfdBirth': 'DATE', + 'dateOfBirth': 'DATE', 'endDate': 'DATE', 'startDate': 'DATE', + 'password': 'password', 'status': 'STRING', - 'landline': 'STRING', + 'profilePicture': 'profilePicture', 'appUserId': 'STRING', - 'Created': 'DATETIME', - 'Updated': 'DATETIME', - 'Created By': 'STRING', - 'Updated By': 'STRING', - 'Assignedgroups': 'ARRAY', - 'Provisionedapps': 'ARRAY', - 'Attributes': 'CUSTOM', - 'Rbacroles': 'ARRAY', - 'Version': 'STRING', - 'Class': 'STRING' + 'landline': 'STRING' } if isinstance(l2, str): From 9065c6393b17a2746085f1e541b4ff152764b83c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 12:18:34 +0530 Subject: [PATCH 30/91] logic added for subset of response --- dev.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/dev.py b/dev.py index e101a81..c48110c 100644 --- a/dev.py +++ b/dev.py @@ -302,9 +302,44 @@ async def get_mapped(data: dict): upsert=True ) + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response" }, + { "$match": { "final_response.value": { "$ne": None } } }, + { "$group": { + "_id": "$final_response.attributeName", + "data": { "$first": "$final_response" } + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value" + }} + ]) + + subset_response_data = list(subset_response) + + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"] + } + json_serializable_response.append(json_serializable_doc) + + logging.debug("Final response saved successfully") - return JSONResponse(content=final_response) + + return JSONResponse(content=json_serializable_response) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) From 2e04f7876106f4014ba902408cdf68b0d060130c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 12:56:32 +0530 Subject: [PATCH 31/91] logic for matching fields to policy --- mapped_data.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 mapped_data.py diff --git a/mapped_data.py b/mapped_data.py new file mode 100644 index 0000000..ec82398 --- /dev/null +++ b/mapped_data.py @@ -0,0 +1,74 @@ +from fastapi import FastAPI, HTTPException +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn + +app = FastAPI() + +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + print(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + # Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + print(f"No match found for '{field}'") + return field, None # Return original field if no match is found + + +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + +@app.post("/generativeaisrvc/map_fields_to_policy/") +async def map_fields_to_policy(payload: Dict[str, Any]): + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body or not policy_mapping: + raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + + mapped_data = {} + + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) From 880a28ecc6fe2676fd1bd74edf579664ec3a097f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 13:17:09 +0530 Subject: [PATCH 32/91] Header added as for tenant --- policy_mapping.py | 194 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 158 insertions(+), 36 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 91d86c4..bba97dd 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -1,5 +1,5 @@ import uvicorn -from fastapi import FastAPI, Form, Request, HTTPException +from fastapi import FastAPI, Form, Request, HTTPException, Header import httpx import json import logging @@ -15,6 +15,9 @@ from datetime import date import datetime import json +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn app = FastAPI() @@ -25,11 +28,11 @@ def stored_input(tenant: str): #logging.debug(f"Getting collection for tenant: {tenant}") - return get_collection(tenant, "input") + return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") - return get_collection(tenant, "output") + return get_collection(tenant, "schema_maker_output") def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' @@ -189,6 +192,7 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat l2_datatype = l2_datatypes.get(element['element_name_l2'], None) final_response.append({ 'jsonPath': matched_data['jsonpath'], + 'attributeName': element['element_name_l1'], 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], 'l2_datatype': l2_datatype, @@ -204,6 +208,7 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat if data['label'] not in processed_labels: final_response.append({ 'jsonPath': data['jsonpath'], + 'attributeName': data['label'], 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 'l2_datatype': '', @@ -212,28 +217,75 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat return final_response +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + print(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + # Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + print(f"No match found for '{field}'") + return field, None # Return original field if no match is found + + +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + +## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(request: Request, data: dict): +async def get_mapped(data: dict, tenant: str = Header(...)): logging.debug(f"API call for auto policy mapping with the application") try: - tenant = "generativeAI" + input_collection = stored_input(tenant) output_collection = stored_response(tenant) - #input_collection.insert_one(data) - input_collection.update_one( - {"appId": data["appId"]}, - {"$set": data}, - upsert=True - ) + # Store the received response directly into the input collection + input_collection.insert_one(data) logging.debug("Input respone saved successfully") print("data :",data) + #appId = data['appId'] + # Assuming the response contains JSON data, you can parse it - json_data = data + + #Start of changes by Abhishek + + json_data = data.get('payload') + + + #End of changes by Abhishek + json_data_ = extract_user_data(json_data) print("json_data: ",json_data_) @@ -249,29 +301,38 @@ async def get_mapped(request: Request, data: dict): l1_list = set(l1) print("list1: ",l1_list) - l2 = ['Id', 'Displayname', 'Firstname', 'Lastname', 'Country', 'Mobile', 'Email', 'Status', 'Created', 'Updated', 'Created By', 'Updated By', 'Assignedgroups', 'Provisionedapps', 'Attributes', 'Rbacroles', 'Version', ' Class'] + + l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { 'Id': 'INTEGER', - 'Displayname': 'STRING', - 'Firstname': 'STRING', - 'Lastname': 'STRING', - 'Country': 'STRING', - 'Mobile': 'STRING', - 'Email': 'STRING', - 'Status': 'STRING', - 'Created': 'DATETIME', - 'Updated': 'DATETIME', - 'Created By': 'STRING', - 'Updated By': 'STRING', - 'Assignedgroups': 'ARRAY', - 'Provisionedapps': 'ARRAY', - 'Attributes': 'CUSTOM', - 'Rbacroles': 'ARRAY', - 'Version': 'STRING', - 'Class': 'STRING' + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'login': 'INTEGER', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'INTEGER', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' } - + if isinstance(l2, str): l2_list = convert_string_to_list(l2) else: @@ -285,21 +346,82 @@ async def get_mapped(request: Request, data: dict): final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response - #appId = data.get("appId") - #final_response_dict['appId'] = appId + appId = data.get("appId") + final_response_dict['appId'] = appId output_collection.update_one( - #{"appId": appId}, + {"appId": appId}, {"$set": final_response_dict}, upsert=True ) + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response" }, + { "$match": { "final_response.value": { "$ne": None } } }, + { "$group": { + "_id": "$final_response.attributeName", + "data": { "$first": "$final_response" } + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value" + }} + ]) + + subset_response_data = list(subset_response) + + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"] + } + json_serializable_response.append(json_serializable_doc) + + logging.debug("Final response saved successfully") - return JSONResponse(content=final_response) + + return JSONResponse(content=json_serializable_response) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/generativeaisrvc/map_fields_to_policy/") +async def map_fields_to_policy(payload: Dict[str, Any]): + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body or not policy_mapping: + raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + + mapped_data = {} + + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=5000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000) From 06197cb2ece83058a7bf32331cebf41673fa2127 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 13:18:10 +0530 Subject: [PATCH 33/91] file name change from dev to policy_mapping --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3144271..6491dea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,4 +21,4 @@ ENV CONFIG_FILE_PATH=/app/config.yaml COPY . /app # Run your FastAPI application -CMD ["uvicorn", "dev:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file +CMD ["uvicorn", "policy_mapping:app", "--host", "0.0.0.0", "--port", "5000"] \ No newline at end of file From 15ed3ea48bafdf7d8cd07e5ebbb40e22bf44eef4 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 13:20:47 +0530 Subject: [PATCH 34/91] routes added --- routes.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/routes.yaml b/routes.yaml index 21c47e4..dbb3576 100644 --- a/routes.yaml +++ b/routes.yaml @@ -30,3 +30,36 @@ spec: version: "" weight: 100 type: ingress-route +--- +api: /v1/config/projects/{project}/routing/ingress/{id} +meta: + id: generativeaiservice + project: unotech +spec: + modify: + headers: [] + outputFormat: yaml + requestTemplate: "" + responseTemplate: "" + template: go + project: unotech + rule: + rule: allow + source: + hosts: + - '*' + methods: + - '*' + port: 0 + rewrite: /generativeaisrvc/map_fields_to_policy + type: prefix + url: /generativeaisrvc/map_fields_to_policy + targets: + - host: generativeaiservice.unotech.svc.cluster.local + port: 8080 + scheme: http + type: "" + version: "" + weight: 100 +type: ingress-route +--- \ No newline at end of file From 88c2c5fa47c2d90a906478e9375570fc169890ac Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 15:11:41 +0530 Subject: [PATCH 35/91] mongo driver unable to handle keys with a . --- policy_mapping.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index bba97dd..2b47b3f 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -262,7 +262,8 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li ## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(data: dict, tenant: str = Header(...)): +async def get_mapped(data: dict, tenant: str = Header(None)): + print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: @@ -270,7 +271,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): output_collection = stored_response(tenant) # Store the received response directly into the input collection - input_collection.insert_one(data) + #input_collection.insert_one(data) logging.debug("Input respone saved successfully") print("data :",data) @@ -283,7 +284,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): json_data = data.get('payload') - + print("json data is {}", json_data) #End of changes by Abhishek json_data_ = extract_user_data(json_data) From e9b74361031d406733f5ff832e04759d18790f19 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 12 Mar 2024 15:20:31 +0530 Subject: [PATCH 36/91] made routes protected --- routes.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/routes.yaml b/routes.yaml index dbb3576..2bf8a4d 100644 --- a/routes.yaml +++ b/routes.yaml @@ -5,14 +5,33 @@ meta: project: unotech spec: modify: - headers: [] + headers: + - key: userid + op: set + value: auth.userId + - key: tenant + op: set + value: auth.tenantId + - key: roles + op: set + value: auth.roles + - key: principal + op: set + value: auth.sub outputFormat: yaml requestTemplate: "" responseTemplate: "" template: go project: unotech rule: - rule: allow + rule: and + clauses: + - rule: authenticated + - rule: match + eval: in + type: string + f2: args.auth.roles + f1: USER source: hosts: - '*' @@ -37,14 +56,33 @@ meta: project: unotech spec: modify: - headers: [] + headers: + - key: userid + op: set + value: auth.userId + - key: tenant + op: set + value: auth.tenantId + - key: roles + op: set + value: auth.roles + - key: principal + op: set + value: auth.sub outputFormat: yaml requestTemplate: "" responseTemplate: "" template: go project: unotech rule: - rule: allow + rule: and + clauses: + - rule: authenticated + - rule: match + eval: in + type: string + f2: args.auth.roles + f1: USER source: hosts: - '*' From 0fac350711e7e5d0003af8723277e8c1b843d43c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 14 Mar 2024 15:16:46 +0530 Subject: [PATCH 37/91] logic added for confidence level --- dev.py | 225 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 188 insertions(+), 37 deletions(-) diff --git a/dev.py b/dev.py index c48110c..e21b471 100644 --- a/dev.py +++ b/dev.py @@ -1,5 +1,5 @@ import uvicorn -from fastapi import FastAPI, Form, Request, HTTPException +from fastapi import FastAPI, Form, Request, HTTPException, Header import httpx import json import logging @@ -15,6 +15,9 @@ from datetime import date import datetime import json +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn app = FastAPI() @@ -24,17 +27,41 @@ ) def stored_input(tenant: str): - #logging.debug(f"Getting collection for tenant: {tenant}") - return get_collection(tenant, "input") + return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): - #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") - return get_collection(tenant, "output") + return get_collection(tenant, "schema_maker_output") + +def stored_subset_response(tenant: str): + return get_collection(tenant, "schema_maker_subset_output") def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] +def stored_score(tenant: str, appId: str): + score_collection = get_collection(tenant, "schema_maker_score") + + # Check if index exists + #index_exists = appId in score_collection.index_information() + + # If index doesn't exist, create it + # if not index_exists: + # score_collection.create_index("appId", unique=True) + confidence_levels = { + "HIGH": [70, 100], + "LOW": [0, 30], + "MEDIUM": [31, 69] + } + # Update or insert a single document for the given appId with confidence levels as fields + score_collection.update_one( + {"appId": appId}, + {"$set": {level: values for level, values in confidence_levels.items()}}, + upsert=True + ) + logging.debug("score collection updated/created successfully") + + return score_collection #-----------------------------extracting the user object from response----------------- @@ -146,9 +173,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): for element_l2 in l2: el1 = str(element_l1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio( - el1, el2 - ) + similarity = fuzz.ratio(el1, el2) if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 @@ -167,74 +192,160 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": element_l2, + "similarity_percentage": similarity_percentage + }) result = {"similar_elements": similar_elements} return result +#----------------------to get the confidence level based on schema_maker_score +def get_confidence_level(similarity_score: float, score_collection) -> str: + + # Query the collection to find the confidence level based on the similarity score + score_doc = score_collection.find_one({ + "$or": [ + {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, + {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, + {"LOW": {"$elemMatch": {"$gte": similarity_score}}} + ] + }) + + # Extract the confidence level based on the matched range + if score_doc: + if similarity_score >= score_doc['HIGH'][0]: + return "HIGH" + elif similarity_score >= score_doc['MEDIUM'][0]: + return "MEDIUM" + elif similarity_score >= score_doc['LOW'][0]: + return "LOW" + else: + return "Unknown" # Should not happen if the schema is properly defined + else: + return "Unknown" # No matching range found, return Unknown + + #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str]) -> List[Dict[str, Union[str, int]]]: +def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str], score_collection) -> List[Dict[str, Union[str, int, float]]]: logging.debug(f"Beautifying the response for saving into the collection") final_response = [] processed_labels = set() - + # Create a dictionary for easy lookup of response_data based on labels response_lookup = {data['label']: data for data in response_data} - for element in similar_elements: # Find matching element in response_data based on label matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) - + if matched_data: l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(element['similarity_percentage'],score_collection) final_response.append({ 'jsonPath': matched_data['jsonpath'], 'attributeName': element['element_name_l1'], 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], 'l2_datatype': l2_datatype, - 'value': matched_data['value'] + 'value': matched_data['value'], + 'similarity_percentage': element['similarity_percentage'], + 'confidence': confidence # Include confidence level }) processed_labels.add(element['element_name_l1']) # Track processed labels - else: print(f"No matched data found for {element['element_name_l1']}") - + + # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(0,score_collection) # Default to 0 for unmatched elements final_response.append({ 'jsonPath': data['jsonpath'], 'attributeName': data['label'], 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 'l2_datatype': '', - 'value': data['value'] # Use value from response_data + 'value': data['value'], # Use value from response_data + 'similarity_percentage': 0, # Default to 0 for unmatched elements + 'confidence': confidence # Include confidence level }) - + return final_response + +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + print(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + # Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + print(f"No match found for '{field}'") + return field, None # Return original field if no match is found + + +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + + +## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(data: dict): +async def get_mapped(data: dict, tenant: str = Header(None)): + print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: - tenant = "generativeAI" + input_collection = stored_input(tenant) output_collection = stored_response(tenant) + subset_collection = stored_subset_response(tenant) # Store the received response directly into the input collection - input_collection.insert_one(data) + #input_collection.insert_one(data) - logging.debug("Input respone saved successfully") - print("data :",data) + #logging.debug("Input respone saved successfully") + #print("data :",data) - - # Assuming the response contains JSON data, you can parse it - json_data = data + json_data = data.get('payload') + + #print("json data is {}", json_data) + #End of changes by Abhishek + json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) + #print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) @@ -289,11 +400,14 @@ async def get_mapped(data: dict): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) + appId = data.get("appId") + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response - appId = data.get("appId") final_response_dict['appId'] = appId output_collection.update_one( @@ -302,12 +416,14 @@ async def get_mapped(data: dict): upsert=True ) + logging.debug("Final response saved successfully") + subset_response = output_collection.aggregate([ - {"$unwind": "$final_response" }, - { "$match": { "final_response.value": { "$ne": None } } }, - { "$group": { + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { "_id": "$final_response.attributeName", - "data": { "$first": "$final_response" } + "data": {"$first": "$final_response"} }}, {"$project": { "_id": 0, @@ -316,12 +432,14 @@ async def get_mapped(data: dict): "l1_datatype": "$data.l1_datatype", "l2_matched": "$data.l2_matched", "l2_datatype": "$data.l2_datatype", - "value": "$data.value" + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence" }} ]) - subset_response_data = list(subset_response) + subset_response_data = list(subset_response) # Serialize each document into a JSON serializable format json_serializable_response = [] for doc in subset_response_data: @@ -331,18 +449,51 @@ async def get_mapped(data: dict): "l1_datatype": doc["l1_datatype"], "l2_matched": doc["l2_matched"], "l2_datatype": doc["l2_datatype"], - "value": doc["value"] + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"] } json_serializable_response.append(json_serializable_doc) - logging.debug("Final response saved successfully") + for data in subset_response_data: + data["appId"] = appId + #print("data: ",data) + + subset_collection.insert_many(subset_response_data) + logging.debug("subset response saved successfully") return JSONResponse(content=json_serializable_response) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/generativeaisrvc/map_fields_to_policy/") +async def map_fields_to_policy(payload: Dict[str, Any]): + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body or not policy_mapping: + raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + + mapped_data = {} + + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000) From 6a85d12c2157ad58a999bcbf92b22e243a8a699c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 14 Mar 2024 15:58:28 +0530 Subject: [PATCH 38/91] logic added for confidence --- policy_mapping.py | 141 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 35 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 2b47b3f..e21b471 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -27,17 +27,41 @@ ) def stored_input(tenant: str): - #logging.debug(f"Getting collection for tenant: {tenant}") return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): - #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") return get_collection(tenant, "schema_maker_output") +def stored_subset_response(tenant: str): + return get_collection(tenant, "schema_maker_subset_output") + def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] +def stored_score(tenant: str, appId: str): + score_collection = get_collection(tenant, "schema_maker_score") + + # Check if index exists + #index_exists = appId in score_collection.index_information() + + # If index doesn't exist, create it + # if not index_exists: + # score_collection.create_index("appId", unique=True) + confidence_levels = { + "HIGH": [70, 100], + "LOW": [0, 30], + "MEDIUM": [31, 69] + } + # Update or insert a single document for the given appId with confidence levels as fields + score_collection.update_one( + {"appId": appId}, + {"$set": {level: values for level, values in confidence_levels.items()}}, + upsert=True + ) + logging.debug("score collection updated/created successfully") + + return score_collection #-----------------------------extracting the user object from response----------------- @@ -149,9 +173,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): for element_l2 in l2: el1 = str(element_l1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio( - el1, el2 - ) + similarity = fuzz.ratio(el1, el2) if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 @@ -170,53 +192,93 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similar_elements.append({"element_name_l1": element_l1, "element_name_l2": element_l2}) + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": element_l2, + "similarity_percentage": similarity_percentage + }) result = {"similar_elements": similar_elements} return result +#----------------------to get the confidence level based on schema_maker_score +def get_confidence_level(similarity_score: float, score_collection) -> str: + + # Query the collection to find the confidence level based on the similarity score + score_doc = score_collection.find_one({ + "$or": [ + {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, + {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, + {"LOW": {"$elemMatch": {"$gte": similarity_score}}} + ] + }) + + # Extract the confidence level based on the matched range + if score_doc: + if similarity_score >= score_doc['HIGH'][0]: + return "HIGH" + elif similarity_score >= score_doc['MEDIUM'][0]: + return "MEDIUM" + elif similarity_score >= score_doc['LOW'][0]: + return "LOW" + else: + return "Unknown" # Should not happen if the schema is properly defined + else: + return "Unknown" # No matching range found, return Unknown + + #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, str]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str]) -> List[Dict[str, Union[str, int]]]: +def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str], score_collection) -> List[Dict[str, Union[str, int, float]]]: logging.debug(f"Beautifying the response for saving into the collection") final_response = [] processed_labels = set() - + # Create a dictionary for easy lookup of response_data based on labels response_lookup = {data['label']: data for data in response_data} - for element in similar_elements: # Find matching element in response_data based on label matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) - + if matched_data: l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(element['similarity_percentage'],score_collection) final_response.append({ 'jsonPath': matched_data['jsonpath'], 'attributeName': element['element_name_l1'], 'l1_datatype': matched_data['datatype'], 'l2_matched': element['element_name_l2'], 'l2_datatype': l2_datatype, - 'value': matched_data['value'] + 'value': matched_data['value'], + 'similarity_percentage': element['similarity_percentage'], + 'confidence': confidence # Include confidence level }) processed_labels.add(element['element_name_l1']) # Track processed labels - else: print(f"No matched data found for {element['element_name_l1']}") - + + # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(0,score_collection) # Default to 0 for unmatched elements final_response.append({ 'jsonPath': data['jsonpath'], 'attributeName': data['label'], 'l1_datatype': data['datatype'], 'l2_matched': '', # No match from l2 'l2_datatype': '', - 'value': data['value'] # Use value from response_data + 'value': data['value'], # Use value from response_data + 'similarity_percentage': 0, # Default to 0 for unmatched elements + 'confidence': confidence # Include confidence level }) - + return final_response + + def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: matched = False # Perform case-insensitive exact match @@ -269,26 +331,21 @@ async def get_mapped(data: dict, tenant: str = Header(None)): input_collection = stored_input(tenant) output_collection = stored_response(tenant) + subset_collection = stored_subset_response(tenant) # Store the received response directly into the input collection #input_collection.insert_one(data) - logging.debug("Input respone saved successfully") - print("data :",data) + #logging.debug("Input respone saved successfully") + #print("data :",data) - #appId = data['appId'] - - # Assuming the response contains JSON data, you can parse it - - #Start of changes by Abhishek - json_data = data.get('payload') - print("json data is {}", json_data) + #print("json data is {}", json_data) #End of changes by Abhishek json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) + #print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) @@ -343,11 +400,14 @@ async def get_mapped(data: dict, tenant: str = Header(None)): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) + appId = data.get("appId") + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response - appId = data.get("appId") final_response_dict['appId'] = appId output_collection.update_one( @@ -356,12 +416,14 @@ async def get_mapped(data: dict, tenant: str = Header(None)): upsert=True ) + logging.debug("Final response saved successfully") + subset_response = output_collection.aggregate([ - {"$unwind": "$final_response" }, - { "$match": { "final_response.value": { "$ne": None } } }, - { "$group": { + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { "_id": "$final_response.attributeName", - "data": { "$first": "$final_response" } + "data": {"$first": "$final_response"} }}, {"$project": { "_id": 0, @@ -370,12 +432,14 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l1_datatype": "$data.l1_datatype", "l2_matched": "$data.l2_matched", "l2_datatype": "$data.l2_datatype", - "value": "$data.value" + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence" }} ]) - subset_response_data = list(subset_response) + subset_response_data = list(subset_response) # Serialize each document into a JSON serializable format json_serializable_response = [] for doc in subset_response_data: @@ -385,13 +449,20 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l1_datatype": doc["l1_datatype"], "l2_matched": doc["l2_matched"], "l2_datatype": doc["l2_datatype"], - "value": doc["value"] + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"] } json_serializable_response.append(json_serializable_doc) - logging.debug("Final response saved successfully") + for data in subset_response_data: + data["appId"] = appId + #print("data: ",data) + + subset_collection.insert_many(subset_response_data) + logging.debug("subset response saved successfully") return JSONResponse(content=json_serializable_response) From 6f8c7317899eaba99363ba9a955bd82dd715e30c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 15 Mar 2024 18:26:39 +0530 Subject: [PATCH 39/91] updated with requestor_id logic --- dev.py | 56 ++++++++++++++++++++++++++++++++++------------- policy_mapping.py | 56 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/dev.py b/dev.py index e21b471..eaee22c 100644 --- a/dev.py +++ b/dev.py @@ -18,6 +18,7 @@ from typing import Dict, Any, List from fuzzywuzzy import process import uvicorn +import uuid app = FastAPI() @@ -26,18 +27,22 @@ format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) +def ResponseModel(data, message, code=200, error_code=None): + return { + "data": data, + "code": code, + "message": message, + "error_code": error_code + } + def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): - return get_collection(tenant, "schema_maker_output") + return get_collection(tenant, "schema_maker_final_output") -def stored_subset_response(tenant: str): - return get_collection(tenant, "schema_maker_subset_output") - -def convert_string_to_list(input_str: str) -> List[str]: - # Remove leading and trailing whitespaces, and split by ',' - return [element.strip() for element in input_str.strip('[]').split(',')] +def stored_policy_mapped(tenant: str): + return get_collection(tenant, "schema_maker_policyMap") def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -63,6 +68,15 @@ def stored_score(tenant: str, appId: str): return score_collection +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + +#--------generate requestor_id----------- +def generate_request_id(): + id = uuid.uuid1() + return id.hex + #-----------------------------extracting the user object from response----------------- def extract_user_data(response): @@ -331,7 +345,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): input_collection = stored_input(tenant) output_collection = stored_response(tenant) - subset_collection = stored_subset_response(tenant) + subset_collection = stored_policy_mapped(tenant) # Store the received response directly into the input collection #input_collection.insert_one(data) @@ -360,10 +374,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): print("list1: ",l1_list) - l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { - 'Id': 'INTEGER', 'department': 'STRING', 'employeeId': 'STRING', 'designation': 'STRING', @@ -401,6 +414,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) appId = data.get("appId") + + requestor_id = generate_request_id() + score_collection = stored_score(tenant, appId) @@ -416,6 +432,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): upsert=True ) + logging.debug("Final response saved successfully") subset_response = output_collection.aggregate([ @@ -456,15 +473,24 @@ async def get_mapped(data: dict, tenant: str = Header(None)): json_serializable_response.append(json_serializable_doc) - for data in subset_response_data: - data["appId"] = appId - #print("data: ",data) + aggregated_data = { + "appId": appId, + "requestor_id": requestor_id, + "subset_data": subset_response_data + } - subset_collection.insert_many(subset_response_data) + subset_collection.insert_one(aggregated_data) logging.debug("subset response saved successfully") - return JSONResponse(content=json_serializable_response) + data_response = { + "requestor_id": requestor_id, + "content": json_serializable_response + } + + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/policy_mapping.py b/policy_mapping.py index e21b471..eaee22c 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -18,6 +18,7 @@ from typing import Dict, Any, List from fuzzywuzzy import process import uvicorn +import uuid app = FastAPI() @@ -26,18 +27,22 @@ format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) +def ResponseModel(data, message, code=200, error_code=None): + return { + "data": data, + "code": code, + "message": message, + "error_code": error_code + } + def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): - return get_collection(tenant, "schema_maker_output") + return get_collection(tenant, "schema_maker_final_output") -def stored_subset_response(tenant: str): - return get_collection(tenant, "schema_maker_subset_output") - -def convert_string_to_list(input_str: str) -> List[str]: - # Remove leading and trailing whitespaces, and split by ',' - return [element.strip() for element in input_str.strip('[]').split(',')] +def stored_policy_mapped(tenant: str): + return get_collection(tenant, "schema_maker_policyMap") def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -63,6 +68,15 @@ def stored_score(tenant: str, appId: str): return score_collection +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + +#--------generate requestor_id----------- +def generate_request_id(): + id = uuid.uuid1() + return id.hex + #-----------------------------extracting the user object from response----------------- def extract_user_data(response): @@ -331,7 +345,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): input_collection = stored_input(tenant) output_collection = stored_response(tenant) - subset_collection = stored_subset_response(tenant) + subset_collection = stored_policy_mapped(tenant) # Store the received response directly into the input collection #input_collection.insert_one(data) @@ -360,10 +374,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): print("list1: ",l1_list) - l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { - 'Id': 'INTEGER', 'department': 'STRING', 'employeeId': 'STRING', 'designation': 'STRING', @@ -401,6 +414,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) appId = data.get("appId") + + requestor_id = generate_request_id() + score_collection = stored_score(tenant, appId) @@ -416,6 +432,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): upsert=True ) + logging.debug("Final response saved successfully") subset_response = output_collection.aggregate([ @@ -456,15 +473,24 @@ async def get_mapped(data: dict, tenant: str = Header(None)): json_serializable_response.append(json_serializable_doc) - for data in subset_response_data: - data["appId"] = appId - #print("data: ",data) + aggregated_data = { + "appId": appId, + "requestor_id": requestor_id, + "subset_data": subset_response_data + } - subset_collection.insert_many(subset_response_data) + subset_collection.insert_one(aggregated_data) logging.debug("subset response saved successfully") - return JSONResponse(content=json_serializable_response) + data_response = { + "requestor_id": requestor_id, + "content": json_serializable_response + } + + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) From 4ab0277716e0fa71e884eff4dde8f9312bbdfe88 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 15 Mar 2024 18:32:47 +0530 Subject: [PATCH 40/91] removed Id mapping for cymmetri and filter on appId --- dev.py | 111 +++++++++++++++++++++++++++++++++++++++------- policy_mapping.py | 15 ++++--- 2 files changed, 104 insertions(+), 22 deletions(-) diff --git a/dev.py b/dev.py index c48110c..13c0cb5 100644 --- a/dev.py +++ b/dev.py @@ -1,5 +1,5 @@ import uvicorn -from fastapi import FastAPI, Form, Request, HTTPException +from fastapi import FastAPI, Form, Request, HTTPException, Header import httpx import json import logging @@ -15,6 +15,9 @@ from datetime import date import datetime import json +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn app = FastAPI() @@ -25,11 +28,11 @@ def stored_input(tenant: str): #logging.debug(f"Getting collection for tenant: {tenant}") - return get_collection(tenant, "input") + return get_collection(tenant, "schema_maker_input") def stored_response(tenant: str): #logging.debug(f"Getting collection for storing scores for tenant: {tenant}") - return get_collection(tenant, "output") + return get_collection(tenant, "schema_maker_output") def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' @@ -214,25 +217,76 @@ def generate_final_response(similar_elements: List[Dict[str, str]], response_dat return final_response +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + print(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + # Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + print(f"No match found for '{field}'") + return field, None # Return original field if no match is found + + +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + +## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(data: dict): +async def get_mapped(data: dict, tenant: str = Header(None)): + print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: - tenant = "generativeAI" + input_collection = stored_input(tenant) output_collection = stored_response(tenant) # Store the received response directly into the input collection - input_collection.insert_one(data) + #input_collection.insert_one(data) logging.debug("Input respone saved successfully") print("data :",data) - + #appId = data['appId'] + # Assuming the response contains JSON data, you can parse it - json_data = data + + #Start of changes by Abhishek + + json_data = data.get('payload') + + print("json data is {}", json_data) + #End of changes by Abhishek + json_data_ = extract_user_data(json_data) print("json_data: ",json_data_) @@ -249,10 +303,9 @@ async def get_mapped(data: dict): print("list1: ",l1_list) - l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { - 'Id': 'INTEGER', 'department': 'STRING', 'employeeId': 'STRING', 'designation': 'STRING', @@ -303,11 +356,11 @@ async def get_mapped(data: dict): ) subset_response = output_collection.aggregate([ - {"$unwind": "$final_response" }, - { "$match": { "final_response.value": { "$ne": None } } }, - { "$group": { + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { "_id": "$final_response.attributeName", - "data": { "$first": "$final_response" } + "data": {"$first": "$final_response"} }}, {"$project": { "_id": 0, @@ -316,7 +369,9 @@ async def get_mapped(data: dict): "l1_datatype": "$data.l1_datatype", "l2_matched": "$data.l2_matched", "l2_datatype": "$data.l2_datatype", - "value": "$data.value" + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence" }} ]) @@ -343,6 +398,32 @@ async def get_mapped(data: dict): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/generativeaisrvc/map_fields_to_policy/") +async def map_fields_to_policy(payload: Dict[str, Any]): + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body or not policy_mapping: + raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + + mapped_data = {} + + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000) diff --git a/policy_mapping.py b/policy_mapping.py index 2b47b3f..13c0cb5 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -303,10 +303,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): print("list1: ",l1_list) - l2 = ['Id','department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { - 'Id': 'INTEGER', 'department': 'STRING', 'employeeId': 'STRING', 'designation': 'STRING', @@ -357,11 +356,11 @@ async def get_mapped(data: dict, tenant: str = Header(None)): ) subset_response = output_collection.aggregate([ - {"$unwind": "$final_response" }, - { "$match": { "final_response.value": { "$ne": None } } }, - { "$group": { + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { "_id": "$final_response.attributeName", - "data": { "$first": "$final_response" } + "data": {"$first": "$final_response"} }}, {"$project": { "_id": 0, @@ -370,7 +369,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l1_datatype": "$data.l1_datatype", "l2_matched": "$data.l2_matched", "l2_datatype": "$data.l2_datatype", - "value": "$data.value" + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence" }} ]) From 01ee27e5075cc37de886339c117bc674c68c6fec Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 18 Mar 2024 19:15:16 +0530 Subject: [PATCH 41/91] added feeback api for request_id for model training --- database/connection.py | 14 +- dev.py | 345 ++++++++++++++++++++++++----------------- policy_mapping.py | 345 ++++++++++++++++++++++++----------------- 3 files changed, 410 insertions(+), 294 deletions(-) diff --git a/database/connection.py b/database/connection.py index ce5cbda..cf7863a 100644 --- a/database/connection.py +++ b/database/connection.py @@ -2,6 +2,8 @@ from pymongo import MongoClient from config.loader import Configuration import platform +from fastapi import FastAPI, Form, Request, HTTPException, Header + def load_configuration(): c = Configuration() @@ -14,7 +16,11 @@ def load_configuration(): mongo_client = MongoClient(MONGO_DETAILS) def get_collection(tenant_name: str, collection_name: str): - mongo_tenant_details = BASE_TENANT_STRING % tenant_name - database = mongo_client[mongo_tenant_details] - generic_collection = database[collection_name] - return generic_collection + try: + mongo_tenant_details = BASE_TENANT_STRING % tenant_name + database = mongo_client[mongo_tenant_details] + generic_collection = database[collection_name] + return generic_collection + except Exception as e: + raise HTTPException(status_code=400, detail="MONGODB_CONNECTION_ERROR") + diff --git a/dev.py b/dev.py index eaee22c..0642c7e 100644 --- a/dev.py +++ b/dev.py @@ -35,6 +35,9 @@ def ResponseModel(data, message, code=200, error_code=None): "error_code": error_code } +def ErrorResponseModel(error, code, message): + return {"error": error, "code": code, "message": message} + def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") @@ -72,103 +75,109 @@ def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] -#--------generate requestor_id----------- +#--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() return id.hex #-----------------------------extracting the user object from response----------------- - def extract_user_data(response): - logging.debug(f"extracting the users from the nested json") - user_data_list = [] - - def is_user_data(obj): - # Check if object contains at least one of the common user data keys - user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} - return any(key in obj for key in user_keys) - - def traverse(obj): - # Recursively traverse the JSON object - nonlocal user_data_list - if isinstance(obj, dict): - if is_user_data(obj): - user_data_list.append(obj) - else: - for value in obj.values(): - traverse(value) - elif isinstance(obj, list): - for item in obj: - traverse(item) + try: + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) - traverse(response) + traverse(response) - return user_data_list + return user_data_list + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + #---------------------------extracting keys, datatype, label and jsonpath---------------- def get_distinct_keys_and_datatypes(json_data): - logging.debug(f"extracting the properties from the json data") - distinct_keys_datatypes = [] - - def explore_json(obj, path=""): - if isinstance(obj, dict): - for key, value in obj.items(): - new_path = f"{path}.{key}" if path else key - if isinstance(value, dict) or isinstance(value, list): - explore_json(value, new_path) - else: - datatype = get_data_type(value) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": key, - "datatype": datatype, - "value": value - }) - elif isinstance(obj, list): - for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) - if isinstance(item, dict): - explore_json(item, new_path) - else: - datatype = get_data_type(item) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": f"Index {index}", - "datatype": datatype, - "value": item - }) - - def get_data_type(value): - if isinstance(value, str): - try: - # Try parsing the value as a date - parse_result = parse(value) - if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): - return 'DATE' # Date if the parsed value matches one of the date formats - else: - if parse_result.time() != datetime.time(0, 0, 0): - return 'DATETIME' + try: + logging.debug(f"extracting the properties from the json data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) else: - return 'STRING' - except (ValueError, OverflowError): - return 'STRING' # Fallback to string if parsing as date/datetime fails - elif isinstance(value, bool): - return 'BOOLEAN' - elif isinstance(value, int): - return 'INTEGER' - elif isinstance(value, float): - return 'FLOAT' - elif isinstance(value, list): - return 'ARRAY' - elif value is None: - return None # Custom type for null values - else: - return 'CUSTOM' + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' - explore_json(json_data) - return distinct_keys_datatypes + explore_json(json_data) + return distinct_keys_datatypes + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #-------------------fuzzy logic matching function---------------------- @@ -218,28 +227,30 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: + try: + # Query the collection to find the confidence level based on the similarity score + score_doc = score_collection.find_one({ + "$or": [ + {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, + {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, + {"LOW": {"$elemMatch": {"$gte": similarity_score}}} + ] + }) - # Query the collection to find the confidence level based on the similarity score - score_doc = score_collection.find_one({ - "$or": [ - {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, - {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, - {"LOW": {"$elemMatch": {"$gte": similarity_score}}} - ] - }) - - # Extract the confidence level based on the matched range - if score_doc: - if similarity_score >= score_doc['HIGH'][0]: - return "HIGH" - elif similarity_score >= score_doc['MEDIUM'][0]: - return "MEDIUM" - elif similarity_score >= score_doc['LOW'][0]: - return "LOW" + # Extract the confidence level based on the matched range + if score_doc: + if similarity_score >= score_doc['HIGH'][0]: + return "HIGH" + elif similarity_score >= score_doc['MEDIUM'][0]: + return "MEDIUM" + elif similarity_score >= score_doc['LOW'][0]: + return "LOW" + else: + return "Unknown" # Should not happen if the schema is properly defined else: - return "Unknown" # Should not happen if the schema is properly defined - else: - return "Unknown" # No matching range found, return Unknown + return "Unknown" # No matching range found, return Unknown + except Exception as e: + raise HTTPException(status_code=400, detail="score_collection_error") #----------------------generates final response--------------- @@ -251,28 +262,29 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], # Create a dictionary for easy lookup of response_data based on labels response_lookup = {data['label']: data for data in response_data} for element in similar_elements: - # Find matching element in response_data based on label - matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) - + matched_data = [data for data in response_data if data['label'] == element['element_name_l1']] + if matched_data: - l2_datatype = l2_datatypes.get(element['element_name_l2'], None) - # Query the schema_maker_score collection to get the confidence level - confidence = get_confidence_level(element['similarity_percentage'],score_collection) - final_response.append({ - 'jsonPath': matched_data['jsonpath'], - 'attributeName': element['element_name_l1'], - 'l1_datatype': matched_data['datatype'], - 'l2_matched': element['element_name_l2'], - 'l2_datatype': l2_datatype, - 'value': matched_data['value'], - 'similarity_percentage': element['similarity_percentage'], - 'confidence': confidence # Include confidence level - }) - processed_labels.add(element['element_name_l1']) # Track processed labels + for match in matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(element['similarity_percentage'], score_collection) + final_response.append({ + 'jsonPath': match['jsonpath'], + 'attributeName': element['element_name_l1'], + 'l1_datatype': match['datatype'], + 'l2_matched': element['element_name_l2'], + 'l2_datatype': l2_datatype, + 'value': match['value'], + 'similarity_percentage': element['similarity_percentage'], + 'confidence': confidence # Include confidence level + }) + processed_labels.add(element['element_name_l1']) # Track processed labels else: print(f"No matched data found for {element['element_name_l1']}") - + print("processed_labels: ",processed_labels) + # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: @@ -349,9 +361,19 @@ async def get_mapped(data: dict, tenant: str = Header(None)): # Store the received response directly into the input collection #input_collection.insert_one(data) - + #logging.debug("Input respone saved successfully") - #print("data :",data) + + # Check if 'appId' and 'payload' are present in the request + if 'appId' not in data: + raise HTTPException(status_code=400, detail="Missing 'appId' in request") + elif 'payload' not in data: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + + # Validate the format of 'payload' + if not isinstance(data['payload'], dict): + raise HTTPException(status_code=400, detail="'payload' must be a dictionary") + json_data = data.get('payload') @@ -363,6 +385,8 @@ async def get_mapped(data: dict, tenant: str = Header(None)): response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) + #print("response_data:", response_data) + l1 = [item['label'] for item in response_data] @@ -412,10 +436,11 @@ async def get_mapped(data: dict, tenant: str = Header(None)): threshold = 60 result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + #print("result: ",result) appId = data.get("appId") - requestor_id = generate_request_id() + request_id = generate_request_id() score_collection = stored_score(tenant, appId) @@ -432,7 +457,6 @@ async def get_mapped(data: dict, tenant: str = Header(None)): upsert=True ) - logging.debug("Final response saved successfully") subset_response = output_collection.aggregate([ @@ -475,7 +499,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): aggregated_data = { "appId": appId, - "requestor_id": requestor_id, + "request_id": request_id, "subset_data": subset_response_data } @@ -484,42 +508,73 @@ async def get_mapped(data: dict, tenant: str = Header(None)): logging.debug("subset response saved successfully") data_response = { - "requestor_id": requestor_id, + "request_id": request_id, "content": json_serializable_response } #return JSONResponse(content=json_serializable_response) return ResponseModel(data=data_response, message="Policy mapping generated successfully") - + + except HTTPException: + raise except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") @app.post("/generativeaisrvc/map_fields_to_policy/") async def map_fields_to_policy(payload: Dict[str, Any]): - body = payload.get("body") - policy_mapping = payload.get("policyMapping") + try: + body = payload.get("body") + policy_mapping = payload.get("policyMapping") - if not body or not policy_mapping: - raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + if not body: + raise HTTPException(status_code=400, detail="body empty") + elif not policy_mapping: + raise HTTPException(status_code=400, detail="policy_mapping empty") - mapped_data = {} + mapped_data = {} - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - mapped_data[field] = value + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + + except HTTPException: + raise + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + + +# @app.get('/query_requestor_id') +# async def query_requestor_id(requestor_id: str, tenant: str = Header(None)): +# if not requestor_id: +# raise HTTPException(status_code=400, detail="requestor_id missing") + +# subset_collection = stored_policy_mapped(tenant) + +# #Implement the policyMaptenant collection data coming from shivani + + +# # Check if requestor_id is present in both collections +# result1 = subset_collection.find_one({'requestor_id': requestor_id}) +# result2 = collection2.find_one({'requestor_id': requestor_id}) + +# if result1 and result2: +# return {"status": "ok"} +# else: +# raise HTTPException(status_code=404, detail="requestor_id not found") - return mapped_data if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index eaee22c..0642c7e 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -35,6 +35,9 @@ def ResponseModel(data, message, code=200, error_code=None): "error_code": error_code } +def ErrorResponseModel(error, code, message): + return {"error": error, "code": code, "message": message} + def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") @@ -72,103 +75,109 @@ def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] -#--------generate requestor_id----------- +#--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() return id.hex #-----------------------------extracting the user object from response----------------- - def extract_user_data(response): - logging.debug(f"extracting the users from the nested json") - user_data_list = [] - - def is_user_data(obj): - # Check if object contains at least one of the common user data keys - user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} - return any(key in obj for key in user_keys) - - def traverse(obj): - # Recursively traverse the JSON object - nonlocal user_data_list - if isinstance(obj, dict): - if is_user_data(obj): - user_data_list.append(obj) - else: - for value in obj.values(): - traverse(value) - elif isinstance(obj, list): - for item in obj: - traverse(item) + try: + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) - traverse(response) + traverse(response) - return user_data_list + return user_data_list + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + #---------------------------extracting keys, datatype, label and jsonpath---------------- def get_distinct_keys_and_datatypes(json_data): - logging.debug(f"extracting the properties from the json data") - distinct_keys_datatypes = [] - - def explore_json(obj, path=""): - if isinstance(obj, dict): - for key, value in obj.items(): - new_path = f"{path}.{key}" if path else key - if isinstance(value, dict) or isinstance(value, list): - explore_json(value, new_path) - else: - datatype = get_data_type(value) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": key, - "datatype": datatype, - "value": value - }) - elif isinstance(obj, list): - for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) - if isinstance(item, dict): - explore_json(item, new_path) - else: - datatype = get_data_type(item) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": f"Index {index}", - "datatype": datatype, - "value": item - }) - - def get_data_type(value): - if isinstance(value, str): - try: - # Try parsing the value as a date - parse_result = parse(value) - if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): - return 'DATE' # Date if the parsed value matches one of the date formats - else: - if parse_result.time() != datetime.time(0, 0, 0): - return 'DATETIME' + try: + logging.debug(f"extracting the properties from the json data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) else: - return 'STRING' - except (ValueError, OverflowError): - return 'STRING' # Fallback to string if parsing as date/datetime fails - elif isinstance(value, bool): - return 'BOOLEAN' - elif isinstance(value, int): - return 'INTEGER' - elif isinstance(value, float): - return 'FLOAT' - elif isinstance(value, list): - return 'ARRAY' - elif value is None: - return None # Custom type for null values - else: - return 'CUSTOM' + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + # Try parsing the value as a date + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' # Date if the parsed value matches one of the date formats + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' # Fallback to string if parsing as date/datetime fails + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None # Custom type for null values + else: + return 'CUSTOM' - explore_json(json_data) - return distinct_keys_datatypes + explore_json(json_data) + return distinct_keys_datatypes + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #-------------------fuzzy logic matching function---------------------- @@ -218,28 +227,30 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: + try: + # Query the collection to find the confidence level based on the similarity score + score_doc = score_collection.find_one({ + "$or": [ + {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, + {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, + {"LOW": {"$elemMatch": {"$gte": similarity_score}}} + ] + }) - # Query the collection to find the confidence level based on the similarity score - score_doc = score_collection.find_one({ - "$or": [ - {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, - {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, - {"LOW": {"$elemMatch": {"$gte": similarity_score}}} - ] - }) - - # Extract the confidence level based on the matched range - if score_doc: - if similarity_score >= score_doc['HIGH'][0]: - return "HIGH" - elif similarity_score >= score_doc['MEDIUM'][0]: - return "MEDIUM" - elif similarity_score >= score_doc['LOW'][0]: - return "LOW" + # Extract the confidence level based on the matched range + if score_doc: + if similarity_score >= score_doc['HIGH'][0]: + return "HIGH" + elif similarity_score >= score_doc['MEDIUM'][0]: + return "MEDIUM" + elif similarity_score >= score_doc['LOW'][0]: + return "LOW" + else: + return "Unknown" # Should not happen if the schema is properly defined else: - return "Unknown" # Should not happen if the schema is properly defined - else: - return "Unknown" # No matching range found, return Unknown + return "Unknown" # No matching range found, return Unknown + except Exception as e: + raise HTTPException(status_code=400, detail="score_collection_error") #----------------------generates final response--------------- @@ -251,28 +262,29 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], # Create a dictionary for easy lookup of response_data based on labels response_lookup = {data['label']: data for data in response_data} for element in similar_elements: - # Find matching element in response_data based on label - matched_data = next((data for data in response_data if data['label'] == element['element_name_l1']), None) - + matched_data = [data for data in response_data if data['label'] == element['element_name_l1']] + if matched_data: - l2_datatype = l2_datatypes.get(element['element_name_l2'], None) - # Query the schema_maker_score collection to get the confidence level - confidence = get_confidence_level(element['similarity_percentage'],score_collection) - final_response.append({ - 'jsonPath': matched_data['jsonpath'], - 'attributeName': element['element_name_l1'], - 'l1_datatype': matched_data['datatype'], - 'l2_matched': element['element_name_l2'], - 'l2_datatype': l2_datatype, - 'value': matched_data['value'], - 'similarity_percentage': element['similarity_percentage'], - 'confidence': confidence # Include confidence level - }) - processed_labels.add(element['element_name_l1']) # Track processed labels + for match in matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + # Query the schema_maker_score collection to get the confidence level + confidence = get_confidence_level(element['similarity_percentage'], score_collection) + final_response.append({ + 'jsonPath': match['jsonpath'], + 'attributeName': element['element_name_l1'], + 'l1_datatype': match['datatype'], + 'l2_matched': element['element_name_l2'], + 'l2_datatype': l2_datatype, + 'value': match['value'], + 'similarity_percentage': element['similarity_percentage'], + 'confidence': confidence # Include confidence level + }) + processed_labels.add(element['element_name_l1']) # Track processed labels else: print(f"No matched data found for {element['element_name_l1']}") - + print("processed_labels: ",processed_labels) + # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: @@ -349,9 +361,19 @@ async def get_mapped(data: dict, tenant: str = Header(None)): # Store the received response directly into the input collection #input_collection.insert_one(data) - + #logging.debug("Input respone saved successfully") - #print("data :",data) + + # Check if 'appId' and 'payload' are present in the request + if 'appId' not in data: + raise HTTPException(status_code=400, detail="Missing 'appId' in request") + elif 'payload' not in data: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + + # Validate the format of 'payload' + if not isinstance(data['payload'], dict): + raise HTTPException(status_code=400, detail="'payload' must be a dictionary") + json_data = data.get('payload') @@ -363,6 +385,8 @@ async def get_mapped(data: dict, tenant: str = Header(None)): response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) + #print("response_data:", response_data) + l1 = [item['label'] for item in response_data] @@ -412,10 +436,11 @@ async def get_mapped(data: dict, tenant: str = Header(None)): threshold = 60 result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) + #print("result: ",result) appId = data.get("appId") - requestor_id = generate_request_id() + request_id = generate_request_id() score_collection = stored_score(tenant, appId) @@ -432,7 +457,6 @@ async def get_mapped(data: dict, tenant: str = Header(None)): upsert=True ) - logging.debug("Final response saved successfully") subset_response = output_collection.aggregate([ @@ -475,7 +499,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): aggregated_data = { "appId": appId, - "requestor_id": requestor_id, + "request_id": request_id, "subset_data": subset_response_data } @@ -484,42 +508,73 @@ async def get_mapped(data: dict, tenant: str = Header(None)): logging.debug("subset response saved successfully") data_response = { - "requestor_id": requestor_id, + "request_id": request_id, "content": json_serializable_response } #return JSONResponse(content=json_serializable_response) return ResponseModel(data=data_response, message="Policy mapping generated successfully") - + + except HTTPException: + raise except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") @app.post("/generativeaisrvc/map_fields_to_policy/") async def map_fields_to_policy(payload: Dict[str, Any]): - body = payload.get("body") - policy_mapping = payload.get("policyMapping") + try: + body = payload.get("body") + policy_mapping = payload.get("policyMapping") - if not body or not policy_mapping: - raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") + if not body: + raise HTTPException(status_code=400, detail="body empty") + elif not policy_mapping: + raise HTTPException(status_code=400, detail="policy_mapping empty") - mapped_data = {} + mapped_data = {} - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - mapped_data[field] = value + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + + except HTTPException: + raise + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + + +# @app.get('/query_requestor_id') +# async def query_requestor_id(requestor_id: str, tenant: str = Header(None)): +# if not requestor_id: +# raise HTTPException(status_code=400, detail="requestor_id missing") + +# subset_collection = stored_policy_mapped(tenant) + +# #Implement the policyMaptenant collection data coming from shivani + + +# # Check if requestor_id is present in both collections +# result1 = subset_collection.find_one({'requestor_id': requestor_id}) +# result2 = collection2.find_one({'requestor_id': requestor_id}) + +# if result1 and result2: +# return {"status": "ok"} +# else: +# raise HTTPException(status_code=404, detail="requestor_id not found") - return mapped_data if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From 7585d31d9bfccafd61c612467836765d88518016 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 19 Mar 2024 11:32:00 +0530 Subject: [PATCH 42/91] logic updated to include all the fields from application --- dev.py | 50 +++++++++++++++++++++++++---------------------- policy_mapping.py | 50 +++++++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/dev.py b/dev.py index 0642c7e..7b20cd5 100644 --- a/dev.py +++ b/dev.py @@ -110,12 +110,10 @@ def traverse(obj): except Exception as e: raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") - #---------------------------extracting keys, datatype, label and jsonpath---------------- - def get_distinct_keys_and_datatypes(json_data): try: - logging.debug(f"extracting the properties from the json data") + logging.debug("Extracting the properties from the JSON data") distinct_keys_datatypes = [] def explore_json(obj, path=""): @@ -133,33 +131,41 @@ def explore_json(obj, path=""): "value": value }) elif isinstance(obj, list): - for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) - if isinstance(item, dict): - explore_json(item, new_path) - else: - datatype = get_data_type(item) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": f"Index {index}", - "datatype": datatype, - "value": item - }) + if not obj: # Check if the list is empty + datatype = 'ARRAY' + distinct_keys_datatypes.append({ + "jsonpath": path, + "label": path.split('.')[-1], # Get the key name from the path + "datatype": datatype, + "value": obj + }) + else: + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict) or isinstance(item, list): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) def get_data_type(value): if isinstance(value, str): try: - # Try parsing the value as a date parse_result = parse(value) if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): - return 'DATE' # Date if the parsed value matches one of the date formats + return 'DATE' else: if parse_result.time() != datetime.time(0, 0, 0): return 'DATETIME' else: return 'STRING' except (ValueError, OverflowError): - return 'STRING' # Fallback to string if parsing as date/datetime fails + return 'STRING' elif isinstance(value, bool): return 'BOOLEAN' elif isinstance(value, int): @@ -169,15 +175,15 @@ def get_data_type(value): elif isinstance(value, list): return 'ARRAY' elif value is None: - return None # Custom type for null values + return None else: return 'CUSTOM' - explore_json(json_data) return distinct_keys_datatypes except Exception as e: - raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + #-------------------fuzzy logic matching function---------------------- @@ -283,8 +289,6 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], else: print(f"No matched data found for {element['element_name_l1']}") - print("processed_labels: ",processed_labels) - # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: diff --git a/policy_mapping.py b/policy_mapping.py index 0642c7e..7b20cd5 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -110,12 +110,10 @@ def traverse(obj): except Exception as e: raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") - #---------------------------extracting keys, datatype, label and jsonpath---------------- - def get_distinct_keys_and_datatypes(json_data): try: - logging.debug(f"extracting the properties from the json data") + logging.debug("Extracting the properties from the JSON data") distinct_keys_datatypes = [] def explore_json(obj, path=""): @@ -133,33 +131,41 @@ def explore_json(obj, path=""): "value": value }) elif isinstance(obj, list): - for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) - if isinstance(item, dict): - explore_json(item, new_path) - else: - datatype = get_data_type(item) - distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": f"Index {index}", - "datatype": datatype, - "value": item - }) + if not obj: # Check if the list is empty + datatype = 'ARRAY' + distinct_keys_datatypes.append({ + "jsonpath": path, + "label": path.split('.')[-1], # Get the key name from the path + "datatype": datatype, + "value": obj + }) + else: + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict) or isinstance(item, list): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) def get_data_type(value): if isinstance(value, str): try: - # Try parsing the value as a date parse_result = parse(value) if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): - return 'DATE' # Date if the parsed value matches one of the date formats + return 'DATE' else: if parse_result.time() != datetime.time(0, 0, 0): return 'DATETIME' else: return 'STRING' except (ValueError, OverflowError): - return 'STRING' # Fallback to string if parsing as date/datetime fails + return 'STRING' elif isinstance(value, bool): return 'BOOLEAN' elif isinstance(value, int): @@ -169,15 +175,15 @@ def get_data_type(value): elif isinstance(value, list): return 'ARRAY' elif value is None: - return None # Custom type for null values + return None else: return 'CUSTOM' - explore_json(json_data) return distinct_keys_datatypes except Exception as e: - raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + #-------------------fuzzy logic matching function---------------------- @@ -283,8 +289,6 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], else: print(f"No matched data found for {element['element_name_l1']}") - print("processed_labels: ",processed_labels) - # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: From 5818b0ef9eb095814dcac16dfbed0c2ca832d88b Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 21 Mar 2024 18:32:15 +0530 Subject: [PATCH 43/91] logic added for custom attributes and synonyms --- dev.py | 193 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 141 insertions(+), 52 deletions(-) diff --git a/dev.py b/dev.py index 7b20cd5..585aa4d 100644 --- a/dev.py +++ b/dev.py @@ -27,6 +27,9 @@ format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) +#------------------- for creating dict--------------- +synonyms_dict = {'department': {'division': 1, 'section': 1, 'unit': 1, 'branch': 1, 'sector': 1, 'departmentalunit': 1, 'team': 1, 'segment': 1, 'group': 1, 'subdivision': 1, 'area': 1, 'office': 1, 'bureau': 1, 'wing': 1, 'part': 1, 'component': 1, 'sphere': 1, 'category': 1, 'subunit': 1, 'subsection': 1}, 'employeeId': {'staffID': 1, 'workerID': 1, 'employeenumber': 1, 'identificationnumber': 1, 'personnelcode': 1, 'staffcode': 1, 'workercode': 1, 'employeereference': 1, 'staffreference': 1, 'personnelnumber': 1, 'employmentID': 1, 'jobID': 1, 'workID': 1, 'staffidentification': 1, 'staffreferencenumber': 1, 'employeeidentifier': 1, 'workeridentifier': 1, 'employmentnumber': 1, 'personnelID': 1, 'personnelidentifier': 1}, 'designation': {'title': 1, 'position': 1, 'role': 1, 'rank': 1, 'jobtitle': 1, 'function': 1, 'appointment': 1, 'capacity': 1, 'post': 1, 'task': 1, 'duty': 1, 'responsibility': 1, 'occupation': 1, 'roletitle': 1, 'positiontitle': 1, 'worktitle': 1, 'jobrole': 1, 'jobposition': 1, 'jobdesignation': 1, 'functiontitle': 1}, 'appUpdatedDate': {'applicationupdatedate': 1, 'softwareupdatedate': 1, 'appmodificationdate': 1, 'updatetimestamp': 1, 'softwarerevisiondate': 1, 'appupgradedate': 1, 'programupdatedate': 1, 'softwarerefreshdate': 1, 'appenhancementdate': 1, 'modificationtimestamp': 1, 'upgradetimestamp': 1, 'revisiontimestamp': 1, 'refreshtimestamp': 1, 'enhancementtimestamp': 1, 'softwarechangedate': 1, 'appchangedate': 1, 'changetimestamp': 1, 'updatetime': 1, 'modificationdate': 1, 'revisiondate': 1}, 'displayName': {'namedisplayed': 1, 'visiblename': 1, 'publicname': 1, 'nameshown': 1, 'presentationname': 1, 'exhibitedname': 1, 'publiclyshownname': 1, 'visibletitle': 1, 'exhibitedtitle': 1, 'publiclyshowntitle': 1, 'presentationtitle': 1, 'showntitle': 1, 'displayedtitle': 1, 'visibledesignation': 1, 'exhibiteddesignation': 1, 'publiclyshowndesignation': 1, 'presentationdesignation': 1, 'showndesignation': 1, 'displayeddesignation': 1, 'displayedidentity': 1}, 'mobile': {'cellular': 1, 'cellphone': 1, 'mobilephone': 1, 'wirelessphone': 1, 'portablephone': 1, 'handset': 1, 'smartphone': 1, 'cell': 1, 'mobiledevice': 1, 'mobilecellular': 1, 'portabledevice': 1, 'wirelessdevice': 1, 'cellulardevice': 1, 'handhelddevice': 1, 'cellularphone': 1, 'cellulartelephone': 1, 'mobileunit': 1, 'wirelessunit': 1}, 'country': {'nation': 1, 'state': 1, 'territory': 1, 'land': 1, 'countrystate': 1, 'nationstate': 1, 'realm': 1, 'region': 1, 'commonwealth': 1, 'province': 1, 'domain': 1, 'sovereignstate': 1, 'nationalterritory': 1, 'nationterritory': 1, 'countryterritory': 1, 'homeland': 1, 'fatherland': 1, 'motherland': 1, 'nativeland': 1, 'soil': 1}, 'city': {'metropolis': 1, 'urbancenter': 1, 'town': 1, 'municipality': 1, 'cityscape': 1, 'borough': 1, 'locality': 1, 'urbanarea': 1, 'downtown': 1, 'community': 1, 'village': 1, 'conurbation': 1, 'township': 1, 'megalopolis': 1, 'cosmopolis': 1, 'megalopolitanarea': 1, 'metropolitanarea': 1, 'megalopolitanregion': 1, 'citycenter': 1, 'citydistrict': 1}, 'email': {'electronicmail': 1, 'emailaddress': 1, 'e-message': 1, 'emailcorrespondence': 1, 'digitalmail': 1, 'e-mail': 1, 'internetmail': 1, 'onlinemail': 1, 'electroniccorrespondence': 1, 'cybermail': 1, 'virtualmail': 1, 'webmail': 1, 'internetmessage': 1, 'e-post': 1, 'e-letter': 1, 'electronicmessage': 1, 'e-communique': 1, 'digitalmessage': 1, 'onlinemessage': 1, 'webmessage': 1}, 'end_date': {'enddate': 1, 'terminationdate': 1, 'conclusiondate': 1, 'finishdate': 1, 'completiondate': 1, 'closingdate': 1, 'expirationdate': 1, 'finaldate': 1, 'culminationdate': 1, 'endingdate': 1, 'expirydate': 1, 'concludingdate': 1, 'ceasingdate': 1, 'lastdate': 1, 'terminationtime': 1, 'conclusiontime': 1, 'finishtime': 1, 'completiontime': 1, 'closingtime': 1, 'expirationtime': 1}, 'firstName': {'givenname': 1, 'forename': 1, 'firstname': 1, 'Christianname': 1, 'personalname': 1, 'individualname': 1, 'giventitle': 1, 'initialname': 1, 'name': 1, 'givenappellation': 1, 'appellation': 1, 'nametag': 1, 'namelabel': 1, 'givendesignation': 1, 'givenidentity': 1, 'title': 1, 'handle': 1, 'moniker': 1, 'nickname': 1, 'nomenclature': 1}, 'login': {'userprincipalname': 1, 'sign-in': 1, 'logon': 1, 'logincredentials': 1, 'access': 1, 'accesscode': 1, 'username': 1, 'userID': 1, 'loginID': 1, 'logonID': 1, 'sign-in details': 1, 'accessdetails': 1, 'accessinformation': 1, 'sign-ininformation': 1, 'logoninformation': 1, 'credentials': 1, 'authentication': 1, 'loginname': 1, 'sign-inname': 1, 'logonname': 1, 'accessname': 1}, 'lastName': {'familyname': 1, 'surname': 1, 'lastname': 1, 'secondname': 1, 'patronymic': 1, 'matronymic': 1, 'sirename': 1, 'maidenname': 1, 'maidensurname': 1, 'parentalname': 1, 'parentalsurname': 1, 'cognomen': 1, 'familytitle': 1, 'familyappellation': 1, 'familylabel': 1, 'familydesignation': 1, 'familyidentity': 1, 'familyhandle': 1}, 'userType': {'roletype': 1, 'usercategory': 1, 'accounttype': 1, 'userrole': 1, 'profiletype': 1, 'identitytype': 1, 'classification': 1, 'userclassification': 1, 'rolecategory': 1, 'userclass': 1, 'identityclass': 1, 'profileclass': 1, 'usergroup': 1, 'identitygroup': 1, 'profilegroup': 1, 'roleclassification': 1, 'userroleclassification': 1, 'identityroleclassification': 1, 'profileroleclassification': 1, 'useridentitytype': 1}, 'dateOfBirth': {'birthdate': 1, 'DOB': 1, 'dateofbirth': 1, 'natalday': 1, 'bornday': 1, 'anniversaryofbirth': 1, 'nativitydate': 1, 'birthday': 1, 'borndate': 1, 'nataldate': 1, 'anniversaryofnativity': 1, 'natalanniversary': 1, 'bornanniversary': 1, 'birthanniversary': 1, 'nativityday': 1, 'birthdayanniversary': 1}, 'endDate': {'enddate': 1, 'conclusiondate': 1, 'terminationdate': 1, 'finishdate': 1, 'completiondate': 1, 'closingdate': 1, 'expirationdate': 1, 'finaldate': 1, 'culminationdate': 1, 'endingdate': 1, 'expirydate': 1, 'concludingdate': 1, 'ceasingdate': 1, 'lastdate': 1, 'terminationtime': 1, 'conclusiontime': 1, 'finishtime': 1, 'completiontime': 1, 'closingtime': 1, 'expirationtime': 1}, 'startDate': {'startdate': 1, 'commencementdate': 1, 'beginningdate': 1, 'initiationdate': 1, 'commencingdate': 1, 'onsetdate': 1, 'commencementtime': 1, 'initiationtime': 1, 'starttime': 1, 'commencingtime': 1, 'onsettime': 1, 'commencementpoint': 1, 'initiationpoint': 1, 'startingpoint': 1, 'commencingpoint': 1, 'onsetpoint': 1, 'launchdate': 1, 'kickoffdate': 1, 'openingdate': 1, 'inaugurationdate': 1}, 'password': {'passcode': 1, 'accesscode': 1, 'securitycode': 1, 'logincode': 1, 'passphrase': 1, 'authenticationcode': 1, 'key': 1, 'secretkey': 1, 'code': 1, 'PIN': 1, 'loginkey': 1, 'accesskey': 1, 'passkey': 1, 'securitykey': 1, 'identificationcode': 1, 'authenticationkey': 1, 'cipher': 1, 'loginpassword': 1, 'securitypassword': 1, 'accesspassword': 1}, 'status': {'condition': 1, 'state': 1, 'situation': 1, 'standing': 1, 'position': 1, 'circumstance': 1, 'statusquo': 1, 'mode': 1, 'stage': 1, 'phase': 1, 'stateofaffairs': 1, 'positioning': 1, 'conditioning': 1, 'stateofbeing': 1, 'statuscondition': 1, 'statusstate': 1, 'statussituation': 1, 'statusposition': 1, 'statuscircumstance': 1, 'statusphase': 1}, 'profilePicture': {'avatar': 1, 'userimage': 1, 'displaypicture': 1, 'profileimage': 1, 'profilephoto': 1, 'userphoto': 1, 'portrait': 1, 'icon': 1, 'thumbnail': 1, 'representation': 1, 'graphic': 1, 'digitalimage': 1, 'visualrepresentation': 1, 'picture': 1, 'photo': 1, 'displayimage': 1, 'profileavatar': 1, 'useravatar': 1, 'image': 1, 'profilerepresentation': 1}, 'appUserId': {'applicationuserID': 1, 'softwareuseridentifier': 1, 'appaccountID': 1, 'userID': 1, 'accountID': 1, 'useridentity': 1, 'usercode': 1, 'useridentifier': 1, 'appidentity': 1, 'softwareID': 1, 'softwareidentifier': 1, 'applicationID': 1, 'applicationidentifier': 1, 'appcode': 1, 'softwarecode': 1, 'accountcode': 1, 'usernumber': 1, 'identitynumber': 1, 'IDcode': 1, 'IDnumber': 1}, 'landline': {'fixedline': 1, 'homephone': 1, 'landlinephone': 1, 'landlinenumber': 1, 'homephonenumber': 1, 'residencephone': 1, 'residenceline': 1, 'telephonenumber': 1, 'fixedphone': 1, 'fixedtelephone': 1, 'domesticphone': 1, 'domesticline': 1, 'domestictelphone': 1, 'housephone': 1, 'houseline': 1, 'housetelephone': 1, 'wiredphone': 1, 'wiredline': 1, 'wiredtelephone': 1, 'cordedphone': 1}} + def ResponseModel(data, message, code=200, error_code=None): return { "data": data, @@ -38,15 +41,51 @@ def ResponseModel(data, message, code=200, error_code=None): def ErrorResponseModel(error, code, message): return {"error": error, "code": code, "message": message} +#--------- stored the payload as input---------- def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") +#--------------stored policymap for all users---------------- def stored_response(tenant: str): return get_collection(tenant, "schema_maker_final_output") +#------------subset policy map response-------------- def stored_policy_mapped(tenant: str): return get_collection(tenant, "schema_maker_policyMap") +#---------synonyms_dict for training-------- +def retreive_synonyms(tenant: str): + return get_collection(tenant, "synonyms_dict") + +#------final policymap by admin for training purpose--------- +def stored_admin_policymap(tenant: str): + return get_collection(tenant, "schema_maker_final_policyMap") + +#----------custom attributes for appending in cymmetri list----- +def retreive_custom_attributes(tenant: str): + return get_collection(tenant, "custome_attribute_master") + +#----------update the synonyms_dict by retraining------------ +def stored_synonyms_dict(tenant: str, appId: str, synonyms_dict: dict): + synonyms_collection = get_collection(tenant, "synonyms_dict") + + # Construct the document to be inserted or updated + document = { + "appId": appId, + "synonyms": {key: [{"synonym": synonym, "score": synonyms_dict[key][synonym]} for synonym in synonyms_dict[key]] for key in synonyms_dict} + } + + # Update the document in the collection + synonyms_collection.update_one( + {"appId": appId}, + {"$set": document}, + upsert=True # This will insert the document if it doesn't exist + ) + logging.debug("synonyms dict collection updated/created successfully") + + return synonyms_collection + +#----------- score for confidence level def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -57,9 +96,9 @@ def stored_score(tenant: str, appId: str): # if not index_exists: # score_collection.create_index("appId", unique=True) confidence_levels = { - "HIGH": [70, 100], - "LOW": [0, 30], - "MEDIUM": [31, 69] + "HIGH": [0.7, 1], + "LOW": [0, 0.3], + "MEDIUM": [0.31, 0.69] } # Update or insert a single document for the given appId with confidence levels as fields score_collection.update_one( @@ -71,6 +110,7 @@ def stored_score(tenant: str, appId: str): return score_collection +#------ for preprocess purpose----------- def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] @@ -80,6 +120,33 @@ def generate_request_id(): id = uuid.uuid1() return id.hex +#--------------for adding custome attributes in cymmetri field list------------ +def add_custom_attributes_to_list(l2, l2_datatypes, tenant): + + attribute_collection = retreive_custom_attributes(tenant) + + # Query to fetch custom attributes + query_result = attribute_collection.find({"attributeType": "USER", "status": True}) + + logging.debug("query executed succesfully") + + # Loop through the query results + for result in query_result: + # Get the 'name' field from the query result + custom_attribute_name = result['name'] + + # Get the 'provAttributeType' field from the query result + custom_attribute_type = result['provAttributeType'] + + # Append custom attribute name to l2 if it's not already in the list + if custom_attribute_name not in l2: + l2.append(custom_attribute_name) + + # Add custom attribute to l2_datatypes + l2_datatypes[custom_attribute_name] = custom_attribute_type + + return l2, l2_datatypes + #-----------------------------extracting the user object from response----------------- def extract_user_data(response): try: @@ -186,10 +253,7 @@ def get_data_type(value): #-------------------fuzzy logic matching function---------------------- - -def compare_lists_with_fuzzy(l1, l2, threshold=50): - logging.debug(f"comparing logic for list1 and list2") - +def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection, appId): matching_elements_l1 = [] matching_elements_l2 = [] non_matching_elements_l1 = [] @@ -198,7 +262,9 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): for element_l1 in l1: max_similarity = 0 matching_element_l2 = '' - + is_synonym_match = False # Flag to track if a synonym match is found + + # Check similarity with original list (l2) for element_l2 in l2: el1 = str(element_l1).lower() el2 = str(element_l2).lower() @@ -206,10 +272,32 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 - + + # If no match is found in the original list, check synonyms + if not matching_element_l2: + synonyms_doc = synonyms_collection.find_one({"appId": appId}) + if synonyms_doc: + synonyms = synonyms_doc.get("synonyms", {}) + for key, values in synonyms.items(): + for synonym_obj in values: + synonym = synonym_obj.get("synonym", "").lower() + score = synonym_obj.get("score", 0) + adjusted_score = score * 100 + if el1 == synonym and adjusted_score >= threshold: + matching_element_l2 = key + max_similarity = adjusted_score # Use score as similarity percentage + is_synonym_match = True + break + if is_synonym_match: + break + if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) matching_elements_l2.append(matching_element_l2.strip("'")) + if is_synonym_match: + print(f"Match found for '{element_l1}' with synonym '{matching_element_l2}' (Score: {max_similarity})") + else: + print(f"Match found for '{element_l1}' with '{matching_element_l2}' (Similarity: {max_similarity})") else: non_matching_elements_l1.append(element_l1.strip("'")) @@ -221,16 +309,20 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) / 100 + matching_condition = "Fuzzy" similar_elements.append({ "element_name_l1": element_l1, "element_name_l2": element_l2, - "similarity_percentage": similarity_percentage + "similarity_percentage": similarity_percentage, + "matching_condition": matching_condition # Adding matching condition to the result }) + result = {"similar_elements": similar_elements} return result + #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: try: @@ -245,11 +337,11 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: # Extract the confidence level based on the matched range if score_doc: - if similarity_score >= score_doc['HIGH'][0]: + if similarity_score >= score_doc['HIGH'][0] and similarity_score <= score_doc['HIGH'][1]: return "HIGH" - elif similarity_score >= score_doc['MEDIUM'][0]: + elif similarity_score >= score_doc['MEDIUM'][0] and similarity_score <= score_doc['MEDIUM'][1]: return "MEDIUM" - elif similarity_score >= score_doc['LOW'][0]: + elif similarity_score >= score_doc['LOW'][0] and similarity_score <= score_doc['LOW'][1]: return "LOW" else: return "Unknown" # Should not happen if the schema is properly defined @@ -283,7 +375,8 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], 'l2_datatype': l2_datatype, 'value': match['value'], 'similarity_percentage': element['similarity_percentage'], - 'confidence': confidence # Include confidence level + 'confidence': confidence, # Include confidence level + 'matching_condition': element["matching_condition"] }) processed_labels.add(element['element_name_l1']) # Track processed labels else: @@ -302,13 +395,13 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], 'l2_datatype': '', 'value': data['value'], # Use value from response_data 'similarity_percentage': 0, # Default to 0 for unmatched elements - 'confidence': confidence # Include confidence level + 'confidence': confidence, # Include confidence level + "matching_condition": "" }) return final_response - - +#--------------- for mapping the body in body populating api------------------ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: matched = False # Perform case-insensitive exact match @@ -333,7 +426,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str print(f"No match found for '{field}'") return field, None # Return original field if no match is found - +#------- works on nested conditions also def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: mapped_nested_data = {} for field, value in nested_field.items(): @@ -350,11 +443,9 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data - -## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(data: dict, tenant: str = Header(None)): +async def get_mapped(data: dict, tenant: str = Header(...)): print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: @@ -362,7 +453,8 @@ async def get_mapped(data: dict, tenant: str = Header(None)): input_collection = stored_input(tenant) output_collection = stored_response(tenant) subset_collection = stored_policy_mapped(tenant) - + synonyms_collection = retreive_synonyms(tenant) + # Store the received response directly into the input collection #input_collection.insert_one(data) @@ -416,11 +508,11 @@ async def get_mapped(data: dict, tenant: str = Header(None)): 'email': 'STRING', 'end_date': 'DATE', 'firstName': 'STRING', - 'login': 'INTEGER', + 'login': 'STRING', 'lastName': 'STRING', 'userType': 'STRING', 'end_date': 'DATE', - 'login': 'INTEGER', + 'login': 'STRING ', 'userType': 'STRING', 'dateOfBirth': 'DATE', 'endDate': 'DATE', @@ -432,24 +524,29 @@ async def get_mapped(data: dict, tenant: str = Header(None)): 'landline': 'STRING' } + l2, l2_datatypes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + + print("list2: ",l2) + if isinstance(l2, str): l2_list = convert_string_to_list(l2) else: l2_list = l2 threshold = 60 + appId = data.get("appId") - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - #print("result: ",result) + synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) - appId = data.get("appId") + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection, appId) + print("result: ",result) request_id = generate_request_id() - - + score_collection = stored_score(tenant, appId) final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection) + #print("final response: ",final_response) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response @@ -479,7 +576,8 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l2_datatype": "$data.l2_datatype", "value": "$data.value", "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence" + "confidence": "$data.confidence", + "matching_condition": "$data.matching_condition" }} ]) @@ -496,7 +594,8 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l2_datatype": doc["l2_datatype"], "value": doc["value"], "similarity_percentage": doc["similarity_percentage"], - "confidence": doc["confidence"] + "confidence": doc["confidence"], + "matching_condition": doc["matching_condition"] } json_serializable_response.append(json_serializable_doc) @@ -525,7 +624,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - +#------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy/") async def map_fields_to_policy(payload: Dict[str, Any]): try: @@ -558,26 +657,16 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") - -# @app.get('/query_requestor_id') -# async def query_requestor_id(requestor_id: str, tenant: str = Header(None)): -# if not requestor_id: -# raise HTTPException(status_code=400, detail="requestor_id missing") - -# subset_collection = stored_policy_mapped(tenant) - -# #Implement the policyMaptenant collection data coming from shivani - - -# # Check if requestor_id is present in both collections -# result1 = subset_collection.find_one({'requestor_id': requestor_id}) -# result2 = collection2.find_one({'requestor_id': requestor_id}) - -# if result1 and result2: -# return {"status": "ok"} -# else: -# raise HTTPException(status_code=404, detail="requestor_id not found") +#-------------------Api fpr storing the admin final policymap for training purpose----------- +@app.post("/generativeaisrvc/store_data") +async def store_data(payload: dict, tenant: str = Header(None)): + try: + policymap_colection = stored_admin_policymap(tenant) + policymap_colection.insert_one(payload) + return {"message": "Data saved successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": From 4001cdcb075f69b3c9920eb5b9aa3047a0c0ca2c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 21 Mar 2024 18:33:00 +0530 Subject: [PATCH 44/91] logic added for custom attributes --- policy_mapping.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 7b20cd5..acab6e1 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -57,9 +57,9 @@ def stored_score(tenant: str, appId: str): # if not index_exists: # score_collection.create_index("appId", unique=True) confidence_levels = { - "HIGH": [70, 100], - "LOW": [0, 30], - "MEDIUM": [31, 69] + "HIGH": [0.7, 1], + "LOW": [0, 0.3], + "MEDIUM": [0.31, 0.69] } # Update or insert a single document for the given appId with confidence levels as fields score_collection.update_one( @@ -221,7 +221,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower())/ 100 similar_elements.append({ "element_name_l1": element_l1, "element_name_l2": element_l2, @@ -245,14 +245,14 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: # Extract the confidence level based on the matched range if score_doc: - if similarity_score >= score_doc['HIGH'][0]: + if similarity_score >= score_doc['HIGH'][0] and similarity_score <= score_doc['HIGH'][1]: return "HIGH" - elif similarity_score >= score_doc['MEDIUM'][0]: + elif similarity_score >= score_doc['MEDIUM'][0] and similarity_score <= score_doc['MEDIUM'][1]: return "MEDIUM" - elif similarity_score >= score_doc['LOW'][0]: + elif similarity_score >= score_doc['LOW'][0] and similarity_score <= score_doc['LOW'][1]: return "LOW" else: - return "Unknown" # Should not happen if the schema is properly defined + return "Unknown" else: return "Unknown" # No matching range found, return Unknown except Exception as e: From 3951ad17da8cd555e7a750887b550d96412c76c7 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 21 Mar 2024 19:05:49 +0530 Subject: [PATCH 45/91] routes updated for new api which for feedback --- routes.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/routes.yaml b/routes.yaml index 2bf8a4d..5d667f4 100644 --- a/routes.yaml +++ b/routes.yaml @@ -100,4 +100,55 @@ spec: version: "" weight: 100 type: ingress-route +--- +api: /v1/config/projects/{project}/routing/ingress/{id} +meta: + id: generativeaiservice + project: unotech +spec: + modify: + headers: + - key: userid + op: set + value: auth.userId + - key: tenant + op: set + value: auth.tenantId + - key: roles + op: set + value: auth.roles + - key: principal + op: set + value: auth.sub + outputFormat: yaml + requestTemplate: "" + responseTemplate: "" + template: go + project: unotech + rule: + rule: and + clauses: + - rule: authenticated + - rule: match + eval: in + type: string + f2: args.auth.roles + f1: USER + source: + hosts: + - '*' + methods: + - '*' + port: 0 + rewrite: /generativeaisrvc/store_data + type: prefix + url: /generativeaisrvc/store_data + targets: + - host: generativeaiservice.unotech.svc.cluster.local + port: 8080 + scheme: http + type: "" + version: "" + weight: 100 +type: ingress-route --- \ No newline at end of file From f4852723b3f78f980e17d3bb97c804b043e2b8b6 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 21 Mar 2024 19:06:32 +0530 Subject: [PATCH 46/91] api added for feedback --- policy_mapping.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index acab6e1..392fab6 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -47,6 +47,9 @@ def stored_response(tenant: str): def stored_policy_mapped(tenant: str): return get_collection(tenant, "schema_maker_policyMap") +def stored_admin_policymap(tenant: str): + return get_collection(tenant, "schema_maker_final_policyMap") + def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -57,9 +60,9 @@ def stored_score(tenant: str, appId: str): # if not index_exists: # score_collection.create_index("appId", unique=True) confidence_levels = { - "HIGH": [0.7, 1], - "LOW": [0, 0.3], - "MEDIUM": [0.31, 0.69] + "HIGH": [70, 100], + "LOW": [0, 30], + "MEDIUM": [31, 69] } # Update or insert a single document for the given appId with confidence levels as fields score_collection.update_one( @@ -221,7 +224,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): similar_elements = [] for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower())/ 100 + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) similar_elements.append({ "element_name_l1": element_l1, "element_name_l2": element_l2, @@ -558,26 +561,15 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") - -# @app.get('/query_requestor_id') -# async def query_requestor_id(requestor_id: str, tenant: str = Header(None)): -# if not requestor_id: -# raise HTTPException(status_code=400, detail="requestor_id missing") - -# subset_collection = stored_policy_mapped(tenant) - -# #Implement the policyMaptenant collection data coming from shivani - - -# # Check if requestor_id is present in both collections -# result1 = subset_collection.find_one({'requestor_id': requestor_id}) -# result2 = collection2.find_one({'requestor_id': requestor_id}) - -# if result1 and result2: -# return {"status": "ok"} -# else: -# raise HTTPException(status_code=404, detail="requestor_id not found") +@app.post("/generativeaisrvc/store_data") +async def store_data(payload: dict, tenant: str = Header(None)): + try: + policymap_colection = stored_admin_policymap(tenant) + policymap_colection.insert_one(payload) + return {"message": "Data saved successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": From e8870052509f486a52247df973b870aa4a9ae4dc Mon Sep 17 00:00:00 2001 From: Shreyas Date: Sat, 23 Mar 2024 13:04:39 +0530 Subject: [PATCH 47/91] logic update for synonyms matched with aggregation query --- adding_syms.py | 1796 ++++++++++++++++++++++++++++++++++++++++ backup_dev.py | 722 ++++++++++++++++ config/loader.py | 22 - database/connection.py | 9 + dev.py | 195 +++-- policy_mapping.py | 82 +- 6 files changed, 2707 insertions(+), 119 deletions(-) create mode 100644 adding_syms.py create mode 100644 backup_dev.py diff --git a/adding_syms.py b/adding_syms.py new file mode 100644 index 0000000..9d8acdd --- /dev/null +++ b/adding_syms.py @@ -0,0 +1,1796 @@ +import pymongo + +# Connect to MongoDB +client = pymongo.MongoClient("mongodb://unoadmin:devDb123@10.0.1.6:27019,10.0.1.6:27020,10.0.1.6:27021/?authSource=admin&replicaSet=sso-rs&retryWrites=true&w=majority") + +# Select database +db = client["cymmetri"] + +synonyms_dict = { + "department": [ + { + "synonym": "division", + "score": 1 + }, + { + "synonym": "section", + "score": 1 + }, + { + "synonym": "unit", + "score": 1 + }, + { + "synonym": "branch", + "score": 1 + }, + { + "synonym": "sector", + "score": 1 + }, + { + "synonym": "departmentalunit", + "score": 1 + }, + { + "synonym": "team", + "score": 1 + }, + { + "synonym": "segment", + "score": 1 + }, + { + "synonym": "group", + "score": 1 + }, + { + "synonym": "subdivision", + "score": 1 + }, + { + "synonym": "area", + "score": 1 + }, + { + "synonym": "office", + "score": 1 + }, + { + "synonym": "bureau", + "score": 1 + }, + { + "synonym": "wing", + "score": 1 + }, + { + "synonym": "part", + "score": 1 + }, + { + "synonym": "component", + "score": 1 + }, + { + "synonym": "sphere", + "score": 1 + }, + { + "synonym": "category", + "score": 1 + }, + { + "synonym": "subunit", + "score": 1 + }, + { + "synonym": "subsection", + "score": 1 + } + ], + "employeeId": [ + { + "synonym": "staffID", + "score": 1 + }, + { + "synonym": "workerID", + "score": 1 + }, + { + "synonym": "employeenumber", + "score": 1 + }, + { + "synonym": "identificationnumber", + "score": 1 + }, + { + "synonym": "personnelcode", + "score": 1 + }, + { + "synonym": "staffcode", + "score": 1 + }, + { + "synonym": "workercode", + "score": 1 + }, + { + "synonym": "employeereference", + "score": 1 + }, + { + "synonym": "staffreference", + "score": 1 + }, + { + "synonym": "personnelnumber", + "score": 1 + }, + { + "synonym": "employmentID", + "score": 1 + }, + { + "synonym": "jobID", + "score": 1 + }, + { + "synonym": "workID", + "score": 1 + }, + { + "synonym": "staffidentification", + "score": 1 + }, + { + "synonym": "staffreferencenumber", + "score": 1 + }, + { + "synonym": "employeeidentifier", + "score": 1 + }, + { + "synonym": "workeridentifier", + "score": 1 + }, + { + "synonym": "employmentnumber", + "score": 1 + }, + { + "synonym": "personnelID", + "score": 1 + }, + { + "synonym": "personnelidentifier", + "score": 1 + } + ], + "designation": [ + { + "synonym": "title", + "score": 1 + }, + { + "synonym": "position", + "score": 1 + }, + { + "synonym": "role", + "score": 1 + }, + { + "synonym": "rank", + "score": 1 + }, + { + "synonym": "jobtitle", + "score": 1 + }, + { + "synonym": "function", + "score": 1 + }, + { + "synonym": "appointment", + "score": 1 + }, + { + "synonym": "capacity", + "score": 1 + }, + { + "synonym": "post", + "score": 1 + }, + { + "synonym": "task", + "score": 1 + }, + { + "synonym": "duty", + "score": 1 + }, + { + "synonym": "responsibility", + "score": 1 + }, + { + "synonym": "occupation", + "score": 1 + }, + { + "synonym": "roletitle", + "score": 1 + }, + { + "synonym": "positiontitle", + "score": 1 + }, + { + "synonym": "worktitle", + "score": 1 + }, + { + "synonym": "jobrole", + "score": 1 + }, + { + "synonym": "jobposition", + "score": 1 + }, + { + "synonym": "jobdesignation", + "score": 1 + }, + { + "synonym": "functiontitle", + "score": 1 + } + ], + "appUpdatedDate": [ + { + "synonym": "applicationupdatedate", + "score": 1 + }, + { + "synonym": "softwareupdatedate", + "score": 1 + }, + { + "synonym": "appmodificationdate", + "score": 1 + }, + { + "synonym": "updatetimestamp", + "score": 1 + }, + { + "synonym": "softwarerevisiondate", + "score": 1 + }, + { + "synonym": "appupgradedate", + "score": 1 + }, + { + "synonym": "programupdatedate", + "score": 1 + }, + { + "synonym": "softwarerefreshdate", + "score": 1 + }, + { + "synonym": "appenhancementdate", + "score": 1 + }, + { + "synonym": "modificationtimestamp", + "score": 1 + }, + { + "synonym": "upgradetimestamp", + "score": 1 + }, + { + "synonym": "revisiontimestamp", + "score": 1 + }, + { + "synonym": "refreshtimestamp", + "score": 1 + }, + { + "synonym": "enhancementtimestamp", + "score": 1 + }, + { + "synonym": "softwarechangedate", + "score": 1 + }, + { + "synonym": "appchangedate", + "score": 1 + }, + { + "synonym": "changetimestamp", + "score": 1 + }, + { + "synonym": "updatetime", + "score": 1 + }, + { + "synonym": "modificationdate", + "score": 1 + }, + { + "synonym": "revisiondate", + "score": 1 + } + ], + "displayName": [ + { + "synonym": "namedisplayed", + "score": 1 + }, + { + "synonym": "visiblename", + "score": 1 + }, + { + "synonym": "publicname", + "score": 1 + }, + { + "synonym": "nameshown", + "score": 1 + }, + { + "synonym": "presentationname", + "score": 1 + }, + { + "synonym": "exhibitedname", + "score": 1 + }, + { + "synonym": "publiclyshownname", + "score": 1 + }, + { + "synonym": "visibletitle", + "score": 1 + }, + { + "synonym": "exhibitedtitle", + "score": 1 + }, + { + "synonym": "publiclyshowntitle", + "score": 1 + }, + { + "synonym": "presentationtitle", + "score": 1 + }, + { + "synonym": "showntitle", + "score": 1 + }, + { + "synonym": "displayedtitle", + "score": 1 + }, + { + "synonym": "visibledesignation", + "score": 1 + }, + { + "synonym": "exhibiteddesignation", + "score": 1 + }, + { + "synonym": "publiclyshowndesignation", + "score": 1 + }, + { + "synonym": "presentationdesignation", + "score": 1 + }, + { + "synonym": "showndesignation", + "score": 1 + }, + { + "synonym": "displayeddesignation", + "score": 1 + }, + { + "synonym": "displayedidentity", + "score": 1 + } + ], + "mobile": [ + { + "synonym": "cellular", + "score": 1 + }, + { + "synonym": "cellphone", + "score": 1 + }, + { + "synonym": "mobilephone", + "score": 1 + }, + { + "synonym": "wirelessphone", + "score": 1 + }, + { + "synonym": "portablephone", + "score": 1 + }, + { + "synonym": "handset", + "score": 1 + }, + { + "synonym": "smartphone", + "score": 1 + }, + { + "synonym": "cell", + "score": 1 + }, + { + "synonym": "mobiledevice", + "score": 1 + }, + { + "synonym": "mobilecellular", + "score": 1 + }, + { + "synonym": "portabledevice", + "score": 1 + }, + { + "synonym": "wirelessdevice", + "score": 1 + }, + { + "synonym": "cellulardevice", + "score": 1 + }, + { + "synonym": "handhelddevice", + "score": 1 + }, + { + "synonym": "cellularphone", + "score": 1 + }, + { + "synonym": "cellulartelephone", + "score": 1 + }, + { + "synonym": "mobileunit", + "score": 1 + }, + { + "synonym": "wirelessunit", + "score": 1 + } + ], + "country": [ + { + "synonym": "nation", + "score": 1 + }, + { + "synonym": "state", + "score": 1 + }, + { + "synonym": "territory", + "score": 1 + }, + { + "synonym": "land", + "score": 1 + }, + { + "synonym": "countrystate", + "score": 1 + }, + { + "synonym": "nationstate", + "score": 1 + }, + { + "synonym": "realm", + "score": 1 + }, + { + "synonym": "region", + "score": 1 + }, + { + "synonym": "commonwealth", + "score": 1 + }, + { + "synonym": "province", + "score": 1 + }, + { + "synonym": "domain", + "score": 1 + }, + { + "synonym": "sovereignstate", + "score": 1 + }, + { + "synonym": "nationalterritory", + "score": 1 + }, + { + "synonym": "nationterritory", + "score": 1 + }, + { + "synonym": "countryterritory", + "score": 1 + }, + { + "synonym": "homeland", + "score": 1 + }, + { + "synonym": "fatherland", + "score": 1 + }, + { + "synonym": "motherland", + "score": 1 + }, + { + "synonym": "nativeland", + "score": 1 + }, + { + "synonym": "soil", + "score": 1 + } + ], + "city": [ + { + "synonym": "metropolis", + "score": 1 + }, + { + "synonym": "urbancenter", + "score": 1 + }, + { + "synonym": "town", + "score": 1 + }, + { + "synonym": "municipality", + "score": 1 + }, + { + "synonym": "cityscape", + "score": 1 + }, + { + "synonym": "borough", + "score": 1 + }, + { + "synonym": "locality", + "score": 1 + }, + { + "synonym": "urbanarea", + "score": 1 + }, + { + "synonym": "downtown", + "score": 1 + }, + { + "synonym": "community", + "score": 1 + }, + { + "synonym": "village", + "score": 1 + }, + { + "synonym": "conurbation", + "score": 1 + }, + { + "synonym": "township", + "score": 1 + }, + { + "synonym": "megalopolis", + "score": 1 + }, + { + "synonym": "cosmopolis", + "score": 1 + }, + { + "synonym": "megalopolitanarea", + "score": 1 + }, + { + "synonym": "metropolitanarea", + "score": 1 + }, + { + "synonym": "megalopolitanregion", + "score": 1 + }, + { + "synonym": "citycenter", + "score": 1 + }, + { + "synonym": "citydistrict", + "score": 1 + } + ], + "email": [ + { + "synonym": "electronicmail", + "score": 1 + }, + { + "synonym": "emailaddress", + "score": 1 + }, + { + "synonym": "e-message", + "score": 1 + }, + { + "synonym": "emailcorrespondence", + "score": 1 + }, + { + "synonym": "digitalmail", + "score": 1 + }, + { + "synonym": "e-mail", + "score": 1 + }, + { + "synonym": "internetmail", + "score": 1 + }, + { + "synonym": "onlinemail", + "score": 1 + }, + { + "synonym": "electroniccorrespondence", + "score": 1 + }, + { + "synonym": "cybermail", + "score": 1 + }, + { + "synonym": "virtualmail", + "score": 1 + }, + { + "synonym": "webmail", + "score": 1 + }, + { + "synonym": "internetmessage", + "score": 1 + }, + { + "synonym": "e-post", + "score": 1 + }, + { + "synonym": "e-letter", + "score": 1 + }, + { + "synonym": "electronicmessage", + "score": 1 + }, + { + "synonym": "e-communique", + "score": 1 + }, + { + "synonym": "digitalmessage", + "score": 1 + }, + { + "synonym": "onlinemessage", + "score": 1 + }, + { + "synonym": "webmessage", + "score": 1 + } + ], + "end_date": [ + { + "synonym": "enddate", + "score": 1 + }, + { + "synonym": "terminationdate", + "score": 1 + }, + { + "synonym": "conclusiondate", + "score": 1 + }, + { + "synonym": "finishdate", + "score": 1 + }, + { + "synonym": "completiondate", + "score": 1 + }, + { + "synonym": "closingdate", + "score": 1 + }, + { + "synonym": "expirationdate", + "score": 1 + }, + { + "synonym": "finaldate", + "score": 1 + }, + { + "synonym": "culminationdate", + "score": 1 + }, + { + "synonym": "endingdate", + "score": 1 + }, + { + "synonym": "expirydate", + "score": 1 + }, + { + "synonym": "concludingdate", + "score": 1 + }, + { + "synonym": "ceasingdate", + "score": 1 + }, + { + "synonym": "lastdate", + "score": 1 + }, + { + "synonym": "terminationtime", + "score": 1 + }, + { + "synonym": "conclusiontime", + "score": 1 + }, + { + "synonym": "finishtime", + "score": 1 + }, + { + "synonym": "completiontime", + "score": 1 + }, + { + "synonym": "closingtime", + "score": 1 + }, + { + "synonym": "expirationtime", + "score": 1 + } + ], + "firstName": [ + { + "synonym": "givenname", + "score": 1 + }, + { + "synonym": "forename", + "score": 1 + }, + { + "synonym": "firstname", + "score": 1 + }, + { + "synonym": "Christianname", + "score": 1 + }, + { + "synonym": "personalname", + "score": 1 + }, + { + "synonym": "individualname", + "score": 1 + }, + { + "synonym": "giventitle", + "score": 1 + }, + { + "synonym": "initialname", + "score": 1 + }, + { + "synonym": "name", + "score": 1 + }, + { + "synonym": "givenappellation", + "score": 1 + }, + { + "synonym": "appellation", + "score": 1 + }, + { + "synonym": "nametag", + "score": 1 + }, + { + "synonym": "namelabel", + "score": 1 + }, + { + "synonym": "givendesignation", + "score": 1 + }, + { + "synonym": "givenidentity", + "score": 1 + }, + { + "synonym": "title", + "score": 1 + }, + { + "synonym": "handle", + "score": 1 + }, + { + "synonym": "moniker", + "score": 1 + }, + { + "synonym": "nickname", + "score": 1 + }, + { + "synonym": "nomenclature", + "score": 1 + } + ], + "login": [ + { + "synonym": "userprincipalname", + "score": 1 + }, + { + "synonym": "sign-in", + "score": 1 + }, + { + "synonym": "logon", + "score": 1 + }, + { + "synonym": "logincredentials", + "score": 1 + }, + { + "synonym": "access", + "score": 1 + }, + { + "synonym": "accesscode", + "score": 1 + }, + { + "synonym": "username", + "score": 1 + }, + { + "synonym": "userID", + "score": 1 + }, + { + "synonym": "loginID", + "score": 1 + }, + { + "synonym": "logonID", + "score": 1 + }, + { + "synonym": "sign-in details", + "score": 1 + }, + { + "synonym": "accessdetails", + "score": 1 + }, + { + "synonym": "accessinformation", + "score": 1 + }, + { + "synonym": "sign-ininformation", + "score": 1 + }, + { + "synonym": "logoninformation", + "score": 1 + }, + { + "synonym": "credentials", + "score": 1 + }, + { + "synonym": "authentication", + "score": 1 + }, + { + "synonym": "loginname", + "score": 1 + }, + { + "synonym": "sign-inname", + "score": 1 + }, + { + "synonym": "logonname", + "score": 1 + }, + { + "synonym": "accessname", + "score": 1 + } + ], + "lastName": [ + { + "synonym": "familyname", + "score": 1 + }, + { + "synonym": "surname", + "score": 1 + }, + { + "synonym": "lastname", + "score": 1 + }, + { + "synonym": "secondname", + "score": 1 + }, + { + "synonym": "patronymic", + "score": 1 + }, + { + "synonym": "matronymic", + "score": 1 + }, + { + "synonym": "sirename", + "score": 1 + }, + { + "synonym": "maidenname", + "score": 1 + }, + { + "synonym": "maidensurname", + "score": 1 + }, + { + "synonym": "parentalname", + "score": 1 + }, + { + "synonym": "parentalsurname", + "score": 1 + }, + { + "synonym": "cognomen", + "score": 1 + }, + { + "synonym": "familytitle", + "score": 1 + }, + { + "synonym": "familyappellation", + "score": 1 + }, + { + "synonym": "familylabel", + "score": 1 + }, + { + "synonym": "familydesignation", + "score": 1 + }, + { + "synonym": "familyidentity", + "score": 1 + }, + { + "synonym": "familyhandle", + "score": 1 + } + ], + "userType": [ + { + "synonym": "roletype", + "score": 1 + }, + { + "synonym": "usercategory", + "score": 1 + }, + { + "synonym": "accounttype", + "score": 1 + }, + { + "synonym": "userrole", + "score": 1 + }, + { + "synonym": "profiletype", + "score": 1 + }, + { + "synonym": "identitytype", + "score": 1 + }, + { + "synonym": "classification", + "score": 1 + }, + { + "synonym": "userclassification", + "score": 1 + }, + { + "synonym": "rolecategory", + "score": 1 + }, + { + "synonym": "userclass", + "score": 1 + }, + { + "synonym": "identityclass", + "score": 1 + }, + { + "synonym": "profileclass", + "score": 1 + }, + { + "synonym": "usergroup", + "score": 1 + }, + { + "synonym": "identitygroup", + "score": 1 + }, + { + "synonym": "profilegroup", + "score": 1 + }, + { + "synonym": "roleclassification", + "score": 1 + }, + { + "synonym": "userroleclassification", + "score": 1 + }, + { + "synonym": "identityroleclassification", + "score": 1 + }, + { + "synonym": "profileroleclassification", + "score": 1 + }, + { + "synonym": "useridentitytype", + "score": 1 + } + ], + "dateOfBirth": [ + { + "synonym": "birthdate", + "score": 1 + }, + { + "synonym": "DOB", + "score": 1 + }, + { + "synonym": "dateofbirth", + "score": 1 + }, + { + "synonym": "natalday", + "score": 1 + }, + { + "synonym": "bornday", + "score": 1 + }, + { + "synonym": "anniversaryofbirth", + "score": 1 + }, + { + "synonym": "nativitydate", + "score": 1 + }, + { + "synonym": "birthday", + "score": 1 + }, + { + "synonym": "borndate", + "score": 1 + }, + { + "synonym": "nataldate", + "score": 1 + }, + { + "synonym": "anniversaryofnativity", + "score": 1 + }, + { + "synonym": "natalanniversary", + "score": 1 + }, + { + "synonym": "bornanniversary", + "score": 1 + }, + { + "synonym": "birthanniversary", + "score": 1 + }, + { + "synonym": "nativityday", + "score": 1 + }, + { + "synonym": "birthdayanniversary", + "score": 1 + } + ], + "endDate": [ + { + "synonym": "enddate", + "score": 1 + }, + { + "synonym": "conclusiondate", + "score": 1 + }, + { + "synonym": "terminationdate", + "score": 1 + }, + { + "synonym": "finishdate", + "score": 1 + }, + { + "synonym": "completiondate", + "score": 1 + }, + { + "synonym": "closingdate", + "score": 1 + }, + { + "synonym": "expirationdate", + "score": 1 + }, + { + "synonym": "finaldate", + "score": 1 + }, + { + "synonym": "culminationdate", + "score": 1 + }, + { + "synonym": "endingdate", + "score": 1 + }, + { + "synonym": "expirydate", + "score": 1 + }, + { + "synonym": "concludingdate", + "score": 1 + }, + { + "synonym": "ceasingdate", + "score": 1 + }, + { + "synonym": "lastdate", + "score": 1 + }, + { + "synonym": "terminationtime", + "score": 1 + }, + { + "synonym": "conclusiontime", + "score": 1 + }, + { + "synonym": "finishtime", + "score": 1 + }, + { + "synonym": "completiontime", + "score": 1 + }, + { + "synonym": "closingtime", + "score": 1 + }, + { + "synonym": "expirationtime", + "score": 1 + } + ], + "startDate": [ + { + "synonym": "startdate", + "score": 1 + }, + { + "synonym": "commencementdate", + "score": 1 + }, + { + "synonym": "beginningdate", + "score": 1 + }, + { + "synonym": "initiationdate", + "score": 1 + }, + { + "synonym": "commencingdate", + "score": 1 + }, + { + "synonym": "onsetdate", + "score": 1 + }, + { + "synonym": "commencementtime", + "score": 1 + }, + { + "synonym": "initiationtime", + "score": 1 + }, + { + "synonym": "starttime", + "score": 1 + }, + { + "synonym": "commencingtime", + "score": 1 + }, + { + "synonym": "onsettime", + "score": 1 + }, + { + "synonym": "commencementpoint", + "score": 1 + }, + { + "synonym": "initiationpoint", + "score": 1 + }, + { + "synonym": "startingpoint", + "score": 1 + }, + { + "synonym": "commencingpoint", + "score": 1 + }, + { + "synonym": "onsetpoint", + "score": 1 + }, + { + "synonym": "launchdate", + "score": 1 + }, + { + "synonym": "kickoffdate", + "score": 1 + }, + { + "synonym": "openingdate", + "score": 1 + }, + { + "synonym": "inaugurationdate", + "score": 1 + } + ], + "password": [ + { + "synonym": "passcode", + "score": 1 + }, + { + "synonym": "accesscode", + "score": 1 + }, + { + "synonym": "securitycode", + "score": 1 + }, + { + "synonym": "logincode", + "score": 1 + }, + { + "synonym": "passphrase", + "score": 1 + }, + { + "synonym": "authenticationcode", + "score": 1 + }, + { + "synonym": "key", + "score": 1 + }, + { + "synonym": "secretkey", + "score": 1 + }, + { + "synonym": "code", + "score": 1 + }, + { + "synonym": "PIN", + "score": 1 + }, + { + "synonym": "loginkey", + "score": 1 + }, + { + "synonym": "accesskey", + "score": 1 + }, + { + "synonym": "passkey", + "score": 1 + }, + { + "synonym": "securitykey", + "score": 1 + }, + { + "synonym": "identificationcode", + "score": 1 + }, + { + "synonym": "authenticationkey", + "score": 1 + }, + { + "synonym": "cipher", + "score": 1 + }, + { + "synonym": "loginpassword", + "score": 1 + }, + { + "synonym": "securitypassword", + "score": 1 + }, + { + "synonym": "accesspassword", + "score": 1 + } + ], + "status": [ + { + "synonym": "condition", + "score": 1 + }, + { + "synonym": "state", + "score": 1 + }, + { + "synonym": "situation", + "score": 1 + }, + { + "synonym": "standing", + "score": 1 + }, + { + "synonym": "position", + "score": 1 + }, + { + "synonym": "circumstance", + "score": 1 + }, + { + "synonym": "statusquo", + "score": 1 + }, + { + "synonym": "mode", + "score": 1 + }, + { + "synonym": "stage", + "score": 1 + }, + { + "synonym": "phase", + "score": 1 + }, + { + "synonym": "stateofaffairs", + "score": 1 + }, + { + "synonym": "positioning", + "score": 1 + }, + { + "synonym": "conditioning", + "score": 1 + }, + { + "synonym": "stateofbeing", + "score": 1 + }, + { + "synonym": "statuscondition", + "score": 1 + }, + { + "synonym": "statusstate", + "score": 1 + }, + { + "synonym": "statussituation", + "score": 1 + }, + { + "synonym": "statusposition", + "score": 1 + }, + { + "synonym": "statuscircumstance", + "score": 1 + }, + { + "synonym": "statusphase", + "score": 1 + } + ], + "profilePicture": [ + { + "synonym": "avatar", + "score": 1 + }, + { + "synonym": "userimage", + "score": 1 + }, + { + "synonym": "displaypicture", + "score": 1 + }, + { + "synonym": "profileimage", + "score": 1 + }, + { + "synonym": "profilephoto", + "score": 1 + }, + { + "synonym": "userphoto", + "score": 1 + }, + { + "synonym": "portrait", + "score": 1 + }, + { + "synonym": "icon", + "score": 1 + }, + { + "synonym": "thumbnail", + "score": 1 + }, + { + "synonym": "representation", + "score": 1 + }, + { + "synonym": "graphic", + "score": 1 + }, + { + "synonym": "digitalimage", + "score": 1 + }, + { + "synonym": "visualrepresentation", + "score": 1 + }, + { + "synonym": "picture", + "score": 1 + }, + { + "synonym": "photo", + "score": 1 + }, + { + "synonym": "displayimage", + "score": 1 + }, + { + "synonym": "profileavatar", + "score": 1 + }, + { + "synonym": "useravatar", + "score": 1 + }, + { + "synonym": "image", + "score": 1 + }, + { + "synonym": "profilerepresentation", + "score": 1 + } + ], + "appUserId": [ + { + "synonym": "applicationuserID", + "score": 1 + }, + { + "synonym": "softwareuseridentifier", + "score": 1 + }, + { + "synonym": "appaccountID", + "score": 1 + }, + { + "synonym": "userID", + "score": 1 + }, + { + "synonym": "accountID", + "score": 1 + }, + { + "synonym": "useridentity", + "score": 1 + }, + { + "synonym": "usercode", + "score": 1 + }, + { + "synonym": "useridentifier", + "score": 1 + }, + { + "synonym": "appidentity", + "score": 1 + }, + { + "synonym": "softwareID", + "score": 1 + }, + { + "synonym": "softwareidentifier", + "score": 1 + }, + { + "synonym": "applicationID", + "score": 1 + }, + { + "synonym": "applicationidentifier", + "score": 1 + }, + { + "synonym": "appcode", + "score": 1 + }, + { + "synonym": "softwarecode", + "score": 1 + }, + { + "synonym": "accountcode", + "score": 1 + }, + { + "synonym": "usernumber", + "score": 1 + }, + { + "synonym": "identitynumber", + "score": 1 + }, + { + "synonym": "IDcode", + "score": 1 + }, + { + "synonym": "IDnumber", + "score": 1 + } + ], + "landline": [ + { + "synonym": "fixedline", + "score": 1 + }, + { + "synonym": "homephone", + "score": 1 + }, + { + "synonym": "landlinephone", + "score": 1 + }, + { + "synonym": "landlinenumber", + "score": 1 + }, + { + "synonym": "homephonenumber", + "score": 1 + }, + { + "synonym": "residencephone", + "score": 1 + }, + { + "synonym": "residenceline", + "score": 1 + }, + { + "synonym": "telephonenumber", + "score": 1 + }, + { + "synonym": "fixedphone", + "score": 1 + }, + { + "synonym": "fixedtelephone", + "score": 1 + }, + { + "synonym": "domesticphone", + "score": 1 + }, + { + "synonym": "domesticline", + "score": 1 + }, + { + "synonym": "domestictelphone", + "score": 1 + }, + { + "synonym": "housephone", + "score": 1 + }, + { + "synonym": "houseline", + "score": 1 + }, + { + "synonym": "housetelephone", + "score": 1 + }, + { + "synonym": "wiredphone", + "score": 1 + }, + { + "synonym": "wiredline", + "score": 1 + }, + { + "synonym": "wiredtelephone", + "score": 1 + }, + { + "synonym": "cordedphone", + "score": 1 + } + ] + } + +# Select collection (you can create a new collection if it doesn't exist) +collection = db["amayaSynonymsMaster"] + +# Insert the dictionary into the collection +collection.insert_one({"synonyms": synonyms_dict}) + +# Confirm insertion +print("Data inserted successfully.") + diff --git a/backup_dev.py b/backup_dev.py new file mode 100644 index 0000000..bbdb6a0 --- /dev/null +++ b/backup_dev.py @@ -0,0 +1,722 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException, Header +import httpx +import json +import logging +from fastapi import FastAPI, Form +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection, get_master_collection +from dateutil.parser import parse +from datetime import datetime +from datetime import date +import datetime +import json +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn +import uuid +from typing import Set + +app = FastAPI() + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + + +def ResponseModel(data, message, code=200, error_code=None): + return { + "data": data, + "code": code, + "message": message, + "error_code": error_code + } + +def ErrorResponseModel(error, code, message): + return {"error": error, "code": code, "message": message} + +#--------- stored the payload as input---------- +def stored_input(tenant: str): + return get_collection(tenant, "schema_maker_input") + +#--------------stored policymap for all users---------------- +def stored_response(tenant: str): + return get_collection(tenant, "schema_maker_final_output") + +#------------subset policy map response-------------- +def stored_policy_mapped(tenant: str): + return get_collection(tenant, "schema_maker_policyMap") + +#---------synonyms_dict for training-------- +# def retreive_synonyms(tenant: str): +# return get_collection(tenant, "amayaSynonymsMaster") + +#------final policymap by admin for training purpose--------- +def stored_admin_policymap(tenant: str): + return get_collection(tenant, "schema_maker_final_policyMap") + +#----------custom attributes for appending in cymmetri list----- +def retrieve_custom_attributes(tenant: str): + return get_collection(tenant, "custome_attribute_master") + + + +#----------- score for confidence level +def stored_score(tenant: str, appId: str): + score_collection = get_collection(tenant, "schema_maker_score") + + # Check if index exists + #index_exists = appId in score_collection.index_information() + + # If index doesn't exist, create it + # if not index_exists: + # score_collection.create_index("appId", unique=True) + confidence_levels = { + "HIGH": [0.7, 1], + "LOW": [0, 0.3], + "MEDIUM": [0.31, 0.69] + } + # Update or insert a single document for the given appId with confidence levels as fields + score_collection.update_one( + {"appId": appId}, + {"$set": {level: values for level, values in confidence_levels.items()}}, + upsert=True + ) + logging.debug("score collection updated/created successfully") + + return score_collection + +#------ for preprocess purpose----------- +def convert_string_to_list(input_str: str) -> List[str]: + # Remove leading and trailing whitespaces, and split by ',' + return [element.strip() for element in input_str.strip('[]').split(',')] + +#--------generate request_id----------- +def generate_request_id(): + id = uuid.uuid1() + return id.hex + +#--------------for adding custome attributes in cymmetri field list------------ +def add_custom_attributes_to_list(l2, l2_datatypes, tenant): + + attribute_collection = retrieve_custom_attributes(tenant) + + #query_result = attribute_collection.find({"attributeType": "USER", "status": True}) + query_result = attribute_collection.find({"attributeType": "USER", "status": True}) + + + logging.debug("query executed successfully") + + custom_attributes = [] # This will track the names of custom attributes added + + for result in query_result: + custom_attribute_name = result['name'] + custom_attribute_type = result['provAttributeType'] + + if custom_attribute_name not in l2: + l2.append(custom_attribute_name) + custom_attributes.append(custom_attribute_name) # Add to custom_attributes set if it's new + + l2_datatypes[custom_attribute_name] = custom_attribute_type + + return l2, l2_datatypes, custom_attributes + + +#-----------------------------extracting the user object from response----------------- +def extract_user_data(response): + try: + logging.debug(f"extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Check if object contains at least one of the common user data keys + user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} + return any(key in obj for key in user_keys) + + def traverse(obj): + # Recursively traverse the JSON object + nonlocal user_data_list + if isinstance(obj, dict): + if is_user_data(obj): + user_data_list.append(obj) + else: + for value in obj.values(): + traverse(value) + elif isinstance(obj, list): + for item in obj: + traverse(item) + + traverse(response) + + return user_data_list + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + +#---------------------------extracting keys, datatype, label and jsonpath---------------- +def get_distinct_keys_and_datatypes(json_data): + try: + logging.debug("Extracting the properties from the JSON data") + distinct_keys_datatypes = [] + + def explore_json(obj, path=""): + if isinstance(obj, dict): + for key, value in obj.items(): + new_path = f"{path}.{key}" if path else key + if isinstance(value, dict) or isinstance(value, list): + explore_json(value, new_path) + else: + datatype = get_data_type(value) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": key, + "datatype": datatype, + "value": value + }) + elif isinstance(obj, list): + if not obj: # Check if the list is empty + datatype = 'ARRAY' + distinct_keys_datatypes.append({ + "jsonpath": path, + "label": path.split('.')[-1], # Get the key name from the path + "datatype": datatype, + "value": obj + }) + else: + for index, item in enumerate(obj): + new_path = f"{path}.{index}" if path else str(index) + if isinstance(item, dict) or isinstance(item, list): + explore_json(item, new_path) + else: + datatype = get_data_type(item) + distinct_keys_datatypes.append({ + "jsonpath": new_path, + "label": f"Index {index}", + "datatype": datatype, + "value": item + }) + + def get_data_type(value): + if isinstance(value, str): + try: + parse_result = parse(value) + if (parse_result.strftime('%Y-%m-%d') == value) or (parse_result.strftime('%d-%m-%y') == value): + return 'DATE' + else: + if parse_result.time() != datetime.time(0, 0, 0): + return 'DATETIME' + else: + return 'STRING' + except (ValueError, OverflowError): + return 'STRING' + elif isinstance(value, bool): + return 'BOOLEAN' + elif isinstance(value, int): + return 'INTEGER' + elif isinstance(value, float): + return 'FLOAT' + elif isinstance(value, list): + return 'ARRAY' + elif value is None: + return None + else: + return 'CUSTOM' + + explore_json(json_data) + return distinct_keys_datatypes + except Exception as e: + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + + +#-------------------fuzzy logic matching function---------------------- +def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): + matching_elements_l1 = [] + matching_elements_l2 = [] + non_matching_elements_l1 = [] + non_matching_elements_l2 = [] + + for element_l1 in l1: + max_similarity = 0 + matching_element_l2 = '' + is_synonym_match = False # Flag to track if a synonym match is found + + # Check similarity with original list (l2) + for element_l2 in l2: + el1 = str(element_l1).lower() + el2 = str(element_l2).lower() + similarity = fuzz.ratio(el1, el2) + if similarity > max_similarity and similarity >= threshold: + max_similarity = similarity + matching_element_l2 = element_l2 + + # If no match is found in the original list, check synonyms + # if not matching_element_l2: + # synonyms_doc = synonyms_collection.find_one() + # if synonyms_doc: + # synonyms = synonyms_doc.get("synonyms", {}) + # for key, values in synonyms.items(): + # for synonym_obj in values: + # synonym = synonym_obj.get("synonym", "").lower() + # score = synonym_obj.get("score", 0) + # adjusted_score = score * 100 + # if el1 == synonym and adjusted_score >= threshold: + # matching_element_l2 = key + # max_similarity = adjusted_score # Use score as similarity percentage + # is_synonym_match = True + # break + # if is_synonym_match: + # break + + if not matching_element_l2: + synonyms_doc = synonyms_collection.find_one() + if synonyms_doc: + threshold_synonyms = threshold / 100 + data = synonyms_collection.aggregate([ + { + "$project": { + "synonymsArray": { "$objectToArray": "$synonyms" } + } + }, + { "$unwind": "$synonymsArray" }, + { + "$match": { + "synonymsArray.v": { + "$elemMatch": { + "synonym": el1, + "score": { "$gt": threshold_synonyms } + } + } + } + }, + { + "$project": { + "_id": 0, + "key": "$synonymsArray.k", + "synonyms": { + "$filter": { + "input": "$synonymsArray.v", + "as": "syn", + "cond": { + "$and": [ + { "$eq": ["$$syn.synonym", el1] }, + { "$gt": ["$$syn.score", threshold_synonyms] } + ] + } + } + } + } + }, + { "$limit": 1 } # Limiting to one document + ]) + if data: + for document in data: + matching_element_l2 = document.get('key', None) + synonyms = document.get('synonyms', []) + max_similarity = None + if synonyms: + max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) + is_synonym_match = True + break + + else: + matching_element_l2 = None + max_similarity = None + + + if matching_element_l2: + matching_elements_l1.append(element_l1.strip("'")) + matching_elements_l2.append(matching_element_l2.strip("'")) + if is_synonym_match: + print(f"Match found for '{element_l1}' with synonym '{matching_element_l2}' (Score: {max_similarity})") + else: + print(f"Match found for '{element_l1}' with fuzzy '{matching_element_l2}' (Similarity: {max_similarity})") + else: + non_matching_elements_l1.append(element_l1.strip("'")) + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + similar_elements = [] + for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) / 100 + matching_condition = "Fuzzy" + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": element_l2, + "similarity_percentage": similarity_percentage, + "matching_condition": matching_condition # Adding matching condition to the result + }) + + + result = {"similar_elements": similar_elements} + return result + + +#----------------------to get the confidence level based on schema_maker_score +def get_confidence_level(similarity_score: float, score_collection) -> str: + try: + # Query the collection to find the confidence level based on the similarity score + score_doc = score_collection.find_one({ + "$or": [ + {"HIGH": {"$elemMatch": {"$gte": similarity_score}}}, + {"MEDIUM": {"$elemMatch": {"$gte": similarity_score}}}, + {"LOW": {"$elemMatch": {"$gte": similarity_score}}} + ] + }) + + # Extract the confidence level based on the matched range + if score_doc: + if similarity_score >= score_doc['HIGH'][0] and similarity_score <= score_doc['HIGH'][1]: + return "HIGH" + elif similarity_score >= score_doc['MEDIUM'][0] and similarity_score <= score_doc['MEDIUM'][1]: + return "MEDIUM" + elif similarity_score >= score_doc['LOW'][0] and similarity_score <= score_doc['LOW'][1]: + return "LOW" + else: + return "Unknown" # Should not happen if the schema is properly defined + else: + return "Unknown" # No matching range found, return Unknown + except Exception as e: + raise HTTPException(status_code=400, detail="score_collection_error") + + +#----------------------generates final response--------------- +def generate_final_response(similar_elements: List[Dict[str, Union[str, int, float]]], + response_data: List[Dict[str, str]], + l2_datatypes: Dict[str, str], + score_collection, + custom_attributes: List[str]) -> List[Dict[str, Union[str, int, float]]]: + logging.debug("Beautifying the response for saving into the collection") + final_response = [] + + processed_labels = set() + + for element in similar_elements: + matched_data = [data for data in response_data if data['label'] == element['element_name_l1']] + + if matched_data: + for match in matched_data: + l2_datatype = l2_datatypes.get(element['element_name_l2'], None) + confidence = get_confidence_level(element['similarity_percentage'], score_collection) + + # Determine if the attribute is custom based on the existence in the custom attributes set + is_custom = element['element_name_l2'] in custom_attributes + + final_response.append({ + 'jsonPath': match['jsonpath'], + 'attributeName': element['element_name_l1'], + 'l1_datatype': match['datatype'], + 'l2_matched': element['element_name_l2'], + 'l2_datatype': l2_datatype, + 'value': match['value'], + 'similarity_percentage': element['similarity_percentage'], + 'confidence': confidence, + 'matching_condition': element["matching_condition"], + 'isCustom': is_custom + }) + processed_labels.add(element['element_name_l1']) + else: + print(f"No matched data found for {element['element_name_l1']}") + + for data in response_data: + if data['label'] not in processed_labels: + confidence = get_confidence_level(0, score_collection) + final_response.append({ + 'jsonPath': data['jsonpath'], + 'attributeName': data['label'], + 'l1_datatype': data['datatype'], + 'l2_matched': '', + 'l2_datatype': '', + 'value': data['value'], + 'similarity_percentage': 0, + 'confidence': confidence, + 'matching_condition': "", + 'isCustom': False # Explicitly false since there's no l2 match + }) + + return final_response + + +#--------------- for mapping the body in body populating api------------------ +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + print(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + # Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + print(f"No match found for '{field}'") + return field, None # Return original field if no match is found + +#------- works on nested conditions also +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + +#----------------------api for policy mapping----------------------------- +@app.post('/generativeaisrvc/get_policy_mapped') +async def get_mapped(data: dict, tenant: str = Header(...)): + print("Headers:", tenant) + logging.debug(f"API call for auto policy mapping with the application") + try: + + synonyms_collection = get_master_collection("amayaSynonymsMaster") + input_collection = stored_input(tenant) + output_collection = stored_response(tenant) + subset_collection = stored_policy_mapped(tenant) + + custom_attributes = [] + + # Store the received response directly into the input collection + #input_collection.insert_one(data) + + #logging.debug("Input respone saved successfully") + + # Check if 'appId' and 'payload' are present in the request + if 'appId' not in data: + raise HTTPException(status_code=400, detail="Missing 'appId' in request") + elif 'payload' not in data: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + + # Validate the format of 'payload' + if not isinstance(data['payload'], dict): + raise HTTPException(status_code=400, detail="'payload' must be a dictionary") + + + json_data = data.get('payload') + + #print("json data is {}", json_data) + #End of changes by Abhishek + + json_data_ = extract_user_data(json_data) + #print("json_data: ",json_data_) + + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + #print("response_data:", response_data) + + + l1 = [item['label'] for item in response_data] + + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + print("list1: ",l1_list) + else: + l1_list = set(l1) + print("list1: ",l1_list) + + + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + + l2_datatypes = { + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'login': 'STRING', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'STRING ', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' + } + + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + + print("list2: ",l2) + + #print("custom_attributes: ", custom_attributes) + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + + threshold = 60 + appId = data.get("appId") + + #--- Inserting the synonyms_dict for appId-------- + #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) + + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + + request_id = generate_request_id() + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) + #print("final response: ",final_response) + final_response_dict = {"final_response": final_response} + + # Assuming 'appId' is present in the received response + final_response_dict['appId'] = appId + + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { + "_id": "$final_response.attributeName", + "data": {"$first": "$final_response"} + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence", + "matching_condition": "$data.matching_condition", + "isCustom": "$data.isCustom" + }} + ]) + + + subset_response_data = list(subset_response) + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"], + "matching_condition": doc["matching_condition"], + "isCustom": doc["isCustom"] + } + json_serializable_response.append(json_serializable_doc) + + + aggregated_data = { + "appId": appId, + "request_id": request_id, + "policyMapList": subset_response_data + } + + subset_collection.insert_one(aggregated_data) + + logging.debug("subset response saved successfully") + + data_response = { + "request_id": request_id, + "content": json_serializable_response + } + + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + + except HTTPException: + raise + + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") + +#------- Api for body populating---------- +@app.post("/generativeaisrvc/map_fields_to_policy/") +async def map_fields_to_policy(payload: Dict[str, Any]): + try: + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body: + raise HTTPException(status_code=400, detail="body empty") + elif not policy_mapping: + raise HTTPException(status_code=400, detail="policy_mapping empty") + + mapped_data = {} + + for field, value in body.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + return mapped_data + + except HTTPException: + raise + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + + +#-------------------Api fpr storing the admin final policymap for training purpose----------- +@app.post("/generativeaisrvc/store_data") +async def store_data(payload: dict, tenant: str = Header(None)): + try: + policymap_colection = stored_admin_policymap(tenant) + policymap_colection.insert_one(payload) + return {"message": "Data saved successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/config/loader.py b/config/loader.py index c5a6bac..71bfbc6 100644 --- a/config/loader.py +++ b/config/loader.py @@ -2,28 +2,6 @@ import platform import os -# class Configuration: -# def __init__(self, filepath): -# self.data = {} -# with open(filepath, "r") as yamlfile: -# data_loaded = yaml.safe_load(yamlfile) - -# self.data = data_loaded - -# def getConfiguration(self): -# return self.data - -# if platform.system() != 'Windows': -# c = Configuration("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml") -# else: -# #c = Configuration("config.yaml") -# c = Configuration(os.environ.get("/Users/apple/Desktop/cymmetri/cymmetri-microservices-generativeAI/config.yaml", "config.yaml")) - -# ConfObject = c.getConfiguration() - - - - import os class Configuration: diff --git a/database/connection.py b/database/connection.py index cf7863a..c11da2d 100644 --- a/database/connection.py +++ b/database/connection.py @@ -23,4 +23,13 @@ def get_collection(tenant_name: str, collection_name: str): return generic_collection except Exception as e: raise HTTPException(status_code=400, detail="MONGODB_CONNECTION_ERROR") + + +def get_master_collection(collection_name: str): + try: + database = mongo_client['cymmetri'] + master_collection = database[collection_name] + return master_collection + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) # Providing exception details for better debugging diff --git a/dev.py b/dev.py index 585aa4d..273fc9a 100644 --- a/dev.py +++ b/dev.py @@ -9,7 +9,7 @@ from typing import List, Union import uvicorn from typing import List, Dict, Union -from database.connection import get_collection +from database.connection import get_collection, get_master_collection from dateutil.parser import parse from datetime import datetime from datetime import date @@ -19,6 +19,7 @@ from fuzzywuzzy import process import uvicorn import uuid +from typing import Set app = FastAPI() @@ -27,8 +28,6 @@ format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) -#------------------- for creating dict--------------- -synonyms_dict = {'department': {'division': 1, 'section': 1, 'unit': 1, 'branch': 1, 'sector': 1, 'departmentalunit': 1, 'team': 1, 'segment': 1, 'group': 1, 'subdivision': 1, 'area': 1, 'office': 1, 'bureau': 1, 'wing': 1, 'part': 1, 'component': 1, 'sphere': 1, 'category': 1, 'subunit': 1, 'subsection': 1}, 'employeeId': {'staffID': 1, 'workerID': 1, 'employeenumber': 1, 'identificationnumber': 1, 'personnelcode': 1, 'staffcode': 1, 'workercode': 1, 'employeereference': 1, 'staffreference': 1, 'personnelnumber': 1, 'employmentID': 1, 'jobID': 1, 'workID': 1, 'staffidentification': 1, 'staffreferencenumber': 1, 'employeeidentifier': 1, 'workeridentifier': 1, 'employmentnumber': 1, 'personnelID': 1, 'personnelidentifier': 1}, 'designation': {'title': 1, 'position': 1, 'role': 1, 'rank': 1, 'jobtitle': 1, 'function': 1, 'appointment': 1, 'capacity': 1, 'post': 1, 'task': 1, 'duty': 1, 'responsibility': 1, 'occupation': 1, 'roletitle': 1, 'positiontitle': 1, 'worktitle': 1, 'jobrole': 1, 'jobposition': 1, 'jobdesignation': 1, 'functiontitle': 1}, 'appUpdatedDate': {'applicationupdatedate': 1, 'softwareupdatedate': 1, 'appmodificationdate': 1, 'updatetimestamp': 1, 'softwarerevisiondate': 1, 'appupgradedate': 1, 'programupdatedate': 1, 'softwarerefreshdate': 1, 'appenhancementdate': 1, 'modificationtimestamp': 1, 'upgradetimestamp': 1, 'revisiontimestamp': 1, 'refreshtimestamp': 1, 'enhancementtimestamp': 1, 'softwarechangedate': 1, 'appchangedate': 1, 'changetimestamp': 1, 'updatetime': 1, 'modificationdate': 1, 'revisiondate': 1}, 'displayName': {'namedisplayed': 1, 'visiblename': 1, 'publicname': 1, 'nameshown': 1, 'presentationname': 1, 'exhibitedname': 1, 'publiclyshownname': 1, 'visibletitle': 1, 'exhibitedtitle': 1, 'publiclyshowntitle': 1, 'presentationtitle': 1, 'showntitle': 1, 'displayedtitle': 1, 'visibledesignation': 1, 'exhibiteddesignation': 1, 'publiclyshowndesignation': 1, 'presentationdesignation': 1, 'showndesignation': 1, 'displayeddesignation': 1, 'displayedidentity': 1}, 'mobile': {'cellular': 1, 'cellphone': 1, 'mobilephone': 1, 'wirelessphone': 1, 'portablephone': 1, 'handset': 1, 'smartphone': 1, 'cell': 1, 'mobiledevice': 1, 'mobilecellular': 1, 'portabledevice': 1, 'wirelessdevice': 1, 'cellulardevice': 1, 'handhelddevice': 1, 'cellularphone': 1, 'cellulartelephone': 1, 'mobileunit': 1, 'wirelessunit': 1}, 'country': {'nation': 1, 'state': 1, 'territory': 1, 'land': 1, 'countrystate': 1, 'nationstate': 1, 'realm': 1, 'region': 1, 'commonwealth': 1, 'province': 1, 'domain': 1, 'sovereignstate': 1, 'nationalterritory': 1, 'nationterritory': 1, 'countryterritory': 1, 'homeland': 1, 'fatherland': 1, 'motherland': 1, 'nativeland': 1, 'soil': 1}, 'city': {'metropolis': 1, 'urbancenter': 1, 'town': 1, 'municipality': 1, 'cityscape': 1, 'borough': 1, 'locality': 1, 'urbanarea': 1, 'downtown': 1, 'community': 1, 'village': 1, 'conurbation': 1, 'township': 1, 'megalopolis': 1, 'cosmopolis': 1, 'megalopolitanarea': 1, 'metropolitanarea': 1, 'megalopolitanregion': 1, 'citycenter': 1, 'citydistrict': 1}, 'email': {'electronicmail': 1, 'emailaddress': 1, 'e-message': 1, 'emailcorrespondence': 1, 'digitalmail': 1, 'e-mail': 1, 'internetmail': 1, 'onlinemail': 1, 'electroniccorrespondence': 1, 'cybermail': 1, 'virtualmail': 1, 'webmail': 1, 'internetmessage': 1, 'e-post': 1, 'e-letter': 1, 'electronicmessage': 1, 'e-communique': 1, 'digitalmessage': 1, 'onlinemessage': 1, 'webmessage': 1}, 'end_date': {'enddate': 1, 'terminationdate': 1, 'conclusiondate': 1, 'finishdate': 1, 'completiondate': 1, 'closingdate': 1, 'expirationdate': 1, 'finaldate': 1, 'culminationdate': 1, 'endingdate': 1, 'expirydate': 1, 'concludingdate': 1, 'ceasingdate': 1, 'lastdate': 1, 'terminationtime': 1, 'conclusiontime': 1, 'finishtime': 1, 'completiontime': 1, 'closingtime': 1, 'expirationtime': 1}, 'firstName': {'givenname': 1, 'forename': 1, 'firstname': 1, 'Christianname': 1, 'personalname': 1, 'individualname': 1, 'giventitle': 1, 'initialname': 1, 'name': 1, 'givenappellation': 1, 'appellation': 1, 'nametag': 1, 'namelabel': 1, 'givendesignation': 1, 'givenidentity': 1, 'title': 1, 'handle': 1, 'moniker': 1, 'nickname': 1, 'nomenclature': 1}, 'login': {'userprincipalname': 1, 'sign-in': 1, 'logon': 1, 'logincredentials': 1, 'access': 1, 'accesscode': 1, 'username': 1, 'userID': 1, 'loginID': 1, 'logonID': 1, 'sign-in details': 1, 'accessdetails': 1, 'accessinformation': 1, 'sign-ininformation': 1, 'logoninformation': 1, 'credentials': 1, 'authentication': 1, 'loginname': 1, 'sign-inname': 1, 'logonname': 1, 'accessname': 1}, 'lastName': {'familyname': 1, 'surname': 1, 'lastname': 1, 'secondname': 1, 'patronymic': 1, 'matronymic': 1, 'sirename': 1, 'maidenname': 1, 'maidensurname': 1, 'parentalname': 1, 'parentalsurname': 1, 'cognomen': 1, 'familytitle': 1, 'familyappellation': 1, 'familylabel': 1, 'familydesignation': 1, 'familyidentity': 1, 'familyhandle': 1}, 'userType': {'roletype': 1, 'usercategory': 1, 'accounttype': 1, 'userrole': 1, 'profiletype': 1, 'identitytype': 1, 'classification': 1, 'userclassification': 1, 'rolecategory': 1, 'userclass': 1, 'identityclass': 1, 'profileclass': 1, 'usergroup': 1, 'identitygroup': 1, 'profilegroup': 1, 'roleclassification': 1, 'userroleclassification': 1, 'identityroleclassification': 1, 'profileroleclassification': 1, 'useridentitytype': 1}, 'dateOfBirth': {'birthdate': 1, 'DOB': 1, 'dateofbirth': 1, 'natalday': 1, 'bornday': 1, 'anniversaryofbirth': 1, 'nativitydate': 1, 'birthday': 1, 'borndate': 1, 'nataldate': 1, 'anniversaryofnativity': 1, 'natalanniversary': 1, 'bornanniversary': 1, 'birthanniversary': 1, 'nativityday': 1, 'birthdayanniversary': 1}, 'endDate': {'enddate': 1, 'conclusiondate': 1, 'terminationdate': 1, 'finishdate': 1, 'completiondate': 1, 'closingdate': 1, 'expirationdate': 1, 'finaldate': 1, 'culminationdate': 1, 'endingdate': 1, 'expirydate': 1, 'concludingdate': 1, 'ceasingdate': 1, 'lastdate': 1, 'terminationtime': 1, 'conclusiontime': 1, 'finishtime': 1, 'completiontime': 1, 'closingtime': 1, 'expirationtime': 1}, 'startDate': {'startdate': 1, 'commencementdate': 1, 'beginningdate': 1, 'initiationdate': 1, 'commencingdate': 1, 'onsetdate': 1, 'commencementtime': 1, 'initiationtime': 1, 'starttime': 1, 'commencingtime': 1, 'onsettime': 1, 'commencementpoint': 1, 'initiationpoint': 1, 'startingpoint': 1, 'commencingpoint': 1, 'onsetpoint': 1, 'launchdate': 1, 'kickoffdate': 1, 'openingdate': 1, 'inaugurationdate': 1}, 'password': {'passcode': 1, 'accesscode': 1, 'securitycode': 1, 'logincode': 1, 'passphrase': 1, 'authenticationcode': 1, 'key': 1, 'secretkey': 1, 'code': 1, 'PIN': 1, 'loginkey': 1, 'accesskey': 1, 'passkey': 1, 'securitykey': 1, 'identificationcode': 1, 'authenticationkey': 1, 'cipher': 1, 'loginpassword': 1, 'securitypassword': 1, 'accesspassword': 1}, 'status': {'condition': 1, 'state': 1, 'situation': 1, 'standing': 1, 'position': 1, 'circumstance': 1, 'statusquo': 1, 'mode': 1, 'stage': 1, 'phase': 1, 'stateofaffairs': 1, 'positioning': 1, 'conditioning': 1, 'stateofbeing': 1, 'statuscondition': 1, 'statusstate': 1, 'statussituation': 1, 'statusposition': 1, 'statuscircumstance': 1, 'statusphase': 1}, 'profilePicture': {'avatar': 1, 'userimage': 1, 'displaypicture': 1, 'profileimage': 1, 'profilephoto': 1, 'userphoto': 1, 'portrait': 1, 'icon': 1, 'thumbnail': 1, 'representation': 1, 'graphic': 1, 'digitalimage': 1, 'visualrepresentation': 1, 'picture': 1, 'photo': 1, 'displayimage': 1, 'profileavatar': 1, 'useravatar': 1, 'image': 1, 'profilerepresentation': 1}, 'appUserId': {'applicationuserID': 1, 'softwareuseridentifier': 1, 'appaccountID': 1, 'userID': 1, 'accountID': 1, 'useridentity': 1, 'usercode': 1, 'useridentifier': 1, 'appidentity': 1, 'softwareID': 1, 'softwareidentifier': 1, 'applicationID': 1, 'applicationidentifier': 1, 'appcode': 1, 'softwarecode': 1, 'accountcode': 1, 'usernumber': 1, 'identitynumber': 1, 'IDcode': 1, 'IDnumber': 1}, 'landline': {'fixedline': 1, 'homephone': 1, 'landlinephone': 1, 'landlinenumber': 1, 'homephonenumber': 1, 'residencephone': 1, 'residenceline': 1, 'telephonenumber': 1, 'fixedphone': 1, 'fixedtelephone': 1, 'domesticphone': 1, 'domesticline': 1, 'domestictelphone': 1, 'housephone': 1, 'houseline': 1, 'housetelephone': 1, 'wiredphone': 1, 'wiredline': 1, 'wiredtelephone': 1, 'cordedphone': 1}} def ResponseModel(data, message, code=200, error_code=None): return { @@ -54,36 +53,18 @@ def stored_policy_mapped(tenant: str): return get_collection(tenant, "schema_maker_policyMap") #---------synonyms_dict for training-------- -def retreive_synonyms(tenant: str): - return get_collection(tenant, "synonyms_dict") +# def retreive_synonyms(tenant: str): +# return get_collection(tenant, "amayaSynonymsMaster") #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): return get_collection(tenant, "schema_maker_final_policyMap") #----------custom attributes for appending in cymmetri list----- -def retreive_custom_attributes(tenant: str): +def retrieve_custom_attributes(tenant: str): return get_collection(tenant, "custome_attribute_master") -#----------update the synonyms_dict by retraining------------ -def stored_synonyms_dict(tenant: str, appId: str, synonyms_dict: dict): - synonyms_collection = get_collection(tenant, "synonyms_dict") - # Construct the document to be inserted or updated - document = { - "appId": appId, - "synonyms": {key: [{"synonym": synonym, "score": synonyms_dict[key][synonym]} for synonym in synonyms_dict[key]] for key in synonyms_dict} - } - - # Update the document in the collection - synonyms_collection.update_one( - {"appId": appId}, - {"$set": document}, - upsert=True # This will insert the document if it doesn't exist - ) - logging.debug("synonyms dict collection updated/created successfully") - - return synonyms_collection #----------- score for confidence level def stored_score(tenant: str, appId: str): @@ -122,30 +103,29 @@ def generate_request_id(): #--------------for adding custome attributes in cymmetri field list------------ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): - - attribute_collection = retreive_custom_attributes(tenant) - - # Query to fetch custom attributes + + attribute_collection = retrieve_custom_attributes(tenant) + + #query_result = attribute_collection.find({"attributeType": "USER", "status": True}) query_result = attribute_collection.find({"attributeType": "USER", "status": True}) - logging.debug("query executed succesfully") + + logging.debug("query executed successfully") + + custom_attributes = [] # This will track the names of custom attributes added - # Loop through the query results for result in query_result: - # Get the 'name' field from the query result custom_attribute_name = result['name'] - - # Get the 'provAttributeType' field from the query result custom_attribute_type = result['provAttributeType'] - - # Append custom attribute name to l2 if it's not already in the list + if custom_attribute_name not in l2: l2.append(custom_attribute_name) - - # Add custom attribute to l2_datatypes + custom_attributes.append(custom_attribute_name) # Add to custom_attributes set if it's new + l2_datatypes[custom_attribute_name] = custom_attribute_type - return l2, l2_datatypes + return l2, l2_datatypes, custom_attributes + #-----------------------------extracting the user object from response----------------- def extract_user_data(response): @@ -253,7 +233,7 @@ def get_data_type(value): #-------------------fuzzy logic matching function---------------------- -def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection, appId): +def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): matching_elements_l1 = [] matching_elements_l2 = [] non_matching_elements_l1 = [] @@ -273,23 +253,61 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection, appId): max_similarity = similarity matching_element_l2 = element_l2 - # If no match is found in the original list, check synonyms if not matching_element_l2: - synonyms_doc = synonyms_collection.find_one({"appId": appId}) + synonyms_doc = synonyms_collection.find_one() if synonyms_doc: - synonyms = synonyms_doc.get("synonyms", {}) - for key, values in synonyms.items(): - for synonym_obj in values: - synonym = synonym_obj.get("synonym", "").lower() - score = synonym_obj.get("score", 0) - adjusted_score = score * 100 - if el1 == synonym and adjusted_score >= threshold: - matching_element_l2 = key - max_similarity = adjusted_score # Use score as similarity percentage - is_synonym_match = True - break - if is_synonym_match: - break + threshold_synonyms = threshold / 100 + data = synonyms_collection.aggregate([ + { + "$project": { + "synonymsArray": { "$objectToArray": "$synonyms" } + } + }, + { "$unwind": "$synonymsArray" }, + { + "$match": { + "synonymsArray.v": { + "$elemMatch": { + "synonym": el1, + "score": { "$gt": threshold_synonyms } + } + } + } + }, + { + "$project": { + "_id": 0, + "key": "$synonymsArray.k", + "synonyms": { + "$filter": { + "input": "$synonymsArray.v", + "as": "syn", + "cond": { + "$and": [ + { "$eq": ["$$syn.synonym", el1] }, + { "$gt": ["$$syn.score", threshold_synonyms] } + ] + } + } + } + } + }, + { "$limit": 1 } + ]) + if data: + for document in data: + matching_element_l2 = document.get('key', None) + synonyms = document.get('synonyms', []) + max_similarity = None + if synonyms: + max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) + is_synonym_match = True + break + + else: + matching_element_l2 = None + max_similarity = None + if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) @@ -297,7 +315,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection, appId): if is_synonym_match: print(f"Match found for '{element_l1}' with synonym '{matching_element_l2}' (Score: {max_similarity})") else: - print(f"Match found for '{element_l1}' with '{matching_element_l2}' (Similarity: {max_similarity})") + print(f"Match found for '{element_l1}' with fuzzy '{matching_element_l2}' (Similarity: {max_similarity})") else: non_matching_elements_l1.append(element_l1.strip("'")) @@ -352,21 +370,27 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str], score_collection) -> List[Dict[str, Union[str, int, float]]]: - logging.debug(f"Beautifying the response for saving into the collection") +def generate_final_response(similar_elements: List[Dict[str, Union[str, int, float]]], + response_data: List[Dict[str, str]], + l2_datatypes: Dict[str, str], + score_collection, + custom_attributes: List[str]) -> List[Dict[str, Union[str, int, float]]]: + logging.debug("Beautifying the response for saving into the collection") final_response = [] + processed_labels = set() - # Create a dictionary for easy lookup of response_data based on labels - response_lookup = {data['label']: data for data in response_data} for element in similar_elements: matched_data = [data for data in response_data if data['label'] == element['element_name_l1']] if matched_data: for match in matched_data: l2_datatype = l2_datatypes.get(element['element_name_l2'], None) - # Query the schema_maker_score collection to get the confidence level confidence = get_confidence_level(element['similarity_percentage'], score_collection) + + # Determine if the attribute is custom based on the existence in the custom attributes set + is_custom = element['element_name_l2'] in custom_attributes + final_response.append({ 'jsonPath': match['jsonpath'], 'attributeName': element['element_name_l1'], @@ -375,32 +399,33 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], 'l2_datatype': l2_datatype, 'value': match['value'], 'similarity_percentage': element['similarity_percentage'], - 'confidence': confidence, # Include confidence level - 'matching_condition': element["matching_condition"] + 'confidence': confidence, + 'matching_condition': element["matching_condition"], + 'isCustom': is_custom }) - processed_labels.add(element['element_name_l1']) # Track processed labels + processed_labels.add(element['element_name_l1']) else: print(f"No matched data found for {element['element_name_l1']}") - # Handle unmatched elements from l1 for data in response_data: if data['label'] not in processed_labels: - # Query the schema_maker_score collection to get the confidence level - confidence = get_confidence_level(0,score_collection) # Default to 0 for unmatched elements + confidence = get_confidence_level(0, score_collection) final_response.append({ 'jsonPath': data['jsonpath'], 'attributeName': data['label'], 'l1_datatype': data['datatype'], - 'l2_matched': '', # No match from l2 + 'l2_matched': '', 'l2_datatype': '', - 'value': data['value'], # Use value from response_data - 'similarity_percentage': 0, # Default to 0 for unmatched elements - 'confidence': confidence, # Include confidence level - "matching_condition": "" + 'value': data['value'], + 'similarity_percentage': 0, + 'confidence': confidence, + 'matching_condition': "", + 'isCustom': False # Explicitly false since there's no l2 match }) return final_response + #--------------- for mapping the body in body populating api------------------ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: matched = False @@ -449,11 +474,13 @@ async def get_mapped(data: dict, tenant: str = Header(...)): print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: - + + synonyms_collection = get_master_collection("amayaSynonymsMaster") input_collection = stored_input(tenant) output_collection = stored_response(tenant) subset_collection = stored_policy_mapped(tenant) - synonyms_collection = retreive_synonyms(tenant) + + custom_attributes = [] # Store the received response directly into the input collection #input_collection.insert_one(data) @@ -524,9 +551,11 @@ async def get_mapped(data: dict, tenant: str = Header(...)): 'landline': 'STRING' } - l2, l2_datatypes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) print("list2: ",l2) + + #print("custom_attributes: ", custom_attributes) if isinstance(l2, str): l2_list = convert_string_to_list(l2) @@ -536,16 +565,16 @@ async def get_mapped(data: dict, tenant: str = Header(...)): threshold = 60 appId = data.get("appId") - synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) - - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection, appId) - print("result: ",result) + #--- Inserting the synonyms_dict for appId-------- + #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + request_id = generate_request_id() score_collection = stored_score(tenant, appId) - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection) + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) #print("final response: ",final_response) final_response_dict = {"final_response": final_response} @@ -577,7 +606,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): "value": "$data.value", "similarity_percentage": "$data.similarity_percentage", "confidence": "$data.confidence", - "matching_condition": "$data.matching_condition" + "matching_condition": "$data.matching_condition", + "isCustom": "$data.isCustom" }} ]) @@ -595,7 +625,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): "value": doc["value"], "similarity_percentage": doc["similarity_percentage"], "confidence": doc["confidence"], - "matching_condition": doc["matching_condition"] + "matching_condition": doc["matching_condition"], + "isCustom": doc["isCustom"] } json_serializable_response.append(json_serializable_doc) @@ -603,7 +634,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): aggregated_data = { "appId": appId, "request_id": request_id, - "subset_data": subset_response_data + "policyMapList": subset_response_data } subset_collection.insert_one(aggregated_data) diff --git a/policy_mapping.py b/policy_mapping.py index 392fab6..837a3e1 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -190,6 +190,51 @@ def get_data_type(value): #-------------------fuzzy logic matching function---------------------- +# def compare_lists_with_fuzzy(l1, l2, threshold=50): +# logging.debug(f"comparing logic for list1 and list2") + +# matching_elements_l1 = [] +# matching_elements_l2 = [] +# non_matching_elements_l1 = [] +# non_matching_elements_l2 = [] + +# for element_l1 in l1: +# max_similarity = 0 +# matching_element_l2 = '' + +# for element_l2 in l2: +# el1 = str(element_l1).lower() +# el2 = str(element_l2).lower() +# similarity = fuzz.ratio(el1, el2) +# if similarity > max_similarity and similarity >= threshold: +# max_similarity = similarity +# matching_element_l2 = element_l2 + +# if matching_element_l2: +# matching_elements_l1.append(element_l1.strip("'")) +# matching_elements_l2.append(matching_element_l2.strip("'")) +# else: +# non_matching_elements_l1.append(element_l1.strip("'")) + +# non_matching_elements_l2 = [ +# element_l2.strip("'") +# for element_l2 in l2 +# if element_l2.strip("'") not in matching_elements_l2 +# ] + +# similar_elements = [] +# for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): +# similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) +# similar_elements.append({ +# "element_name_l1": element_l1, +# "element_name_l2": element_l2, +# "similarity_percentage": similarity_percentage +# }) + +# result = {"similar_elements": similar_elements} +# return result + + def compare_lists_with_fuzzy(l1, l2, threshold=50): logging.debug(f"comparing logic for list1 and list2") @@ -197,6 +242,7 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): matching_elements_l2 = [] non_matching_elements_l1 = [] non_matching_elements_l2 = [] + similar_elements = [] for element_l1 in l1: max_similarity = 0 @@ -205,10 +251,15 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): for element_l2 in l2: el1 = str(element_l1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio(el1, el2) + similarity = fuzz.ratio(el1, el2)/ 100 if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 + similar_elements.append({ + "element_name_l1": el1, + "element_name_l2": el2, + "similarity_percentage": similarity + }) if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) @@ -216,24 +267,25 @@ def compare_lists_with_fuzzy(l1, l2, threshold=50): else: non_matching_elements_l1.append(element_l1.strip("'")) - non_matching_elements_l2 = [ - element_l2.strip("'") - for element_l2 in l2 - if element_l2.strip("'") not in matching_elements_l2 - ] + # non_matching_elements_l2 = [ + # element_l2.strip("'") + # for element_l2 in l2 + # if element_l2.strip("'") not in matching_elements_l2 + # ] - similar_elements = [] - for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) - similar_elements.append({ - "element_name_l1": element_l1, - "element_name_l2": element_l2, - "similarity_percentage": similarity_percentage - }) + # similar_elements = [] + # for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): + # similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) + # similar_elements.append({ + # "element_name_l1": element_l1, + # "element_name_l2": element_l2, + # "similarity_percentage": similarity_percentage + # }) result = {"similar_elements": similar_elements} return result + #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: try: @@ -443,7 +495,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): threshold = 60 result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - #print("result: ",result) + print("result: ",result) appId = data.get("appId") From 50ae7496ab87100b2095354d102b04c69c0dfdaf Mon Sep 17 00:00:00 2001 From: Shreyas Date: Sat, 23 Mar 2024 14:57:05 +0530 Subject: [PATCH 48/91] fix the issues with percentage for synonyms --- dev.py | 86 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/dev.py b/dev.py index 273fc9a..a572a08 100644 --- a/dev.py +++ b/dev.py @@ -52,9 +52,6 @@ def stored_response(tenant: str): def stored_policy_mapped(tenant: str): return get_collection(tenant, "schema_maker_policyMap") -#---------synonyms_dict for training-------- -# def retreive_synonyms(tenant: str): -# return get_collection(tenant, "amayaSynonymsMaster") #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): @@ -64,8 +61,6 @@ def stored_admin_policymap(tenant: str): def retrieve_custom_attributes(tenant: str): return get_collection(tenant, "custome_attribute_master") - - #----------- score for confidence level def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -238,7 +233,8 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): matching_elements_l2 = [] non_matching_elements_l1 = [] non_matching_elements_l2 = [] - + similar_elements = [] + for element_l1 in l1: max_similarity = 0 matching_element_l2 = '' @@ -256,20 +252,20 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): if not matching_element_l2: synonyms_doc = synonyms_collection.find_one() if synonyms_doc: - threshold_synonyms = threshold / 100 + threshold_synonyms = threshold / 100 data = synonyms_collection.aggregate([ { "$project": { - "synonymsArray": { "$objectToArray": "$synonyms" } + "synonymsArray": {"$objectToArray": "$synonyms"} } }, - { "$unwind": "$synonymsArray" }, + {"$unwind": "$synonymsArray"}, { "$match": { "synonymsArray.v": { "$elemMatch": { "synonym": el1, - "score": { "$gt": threshold_synonyms } + "score": {"$gt": threshold_synonyms} } } } @@ -284,63 +280,64 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): "as": "syn", "cond": { "$and": [ - { "$eq": ["$$syn.synonym", el1] }, - { "$gt": ["$$syn.score", threshold_synonyms] } + {"$eq": ["$$syn.synonym", el1]}, + {"$gt": ["$$syn.score", threshold_synonyms]} ] } } } } }, - { "$limit": 1 } + {"$limit": 1} ]) if data: - for document in data: - matching_element_l2 = document.get('key', None) - synonyms = document.get('synonyms', []) - max_similarity = None - if synonyms: - max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) - is_synonym_match = True - break - + for document in data: + matching_element_l2 = document.get('key', None) + synonyms = document.get('synonyms', []) + max_similarity = None + if synonyms: + max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) + is_synonym_match = True + break + else: matching_element_l2 = None max_similarity = None - if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) matching_elements_l2.append(matching_element_l2.strip("'")) if is_synonym_match: - print(f"Match found for '{element_l1}' with synonym '{matching_element_l2}' (Score: {max_similarity})") + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": max_similarity, + "matching_decision": "synonyms" + }) else: - print(f"Match found for '{element_l1}' with fuzzy '{matching_element_l2}' (Similarity: {max_similarity})") + similarity_percentage = fuzz.ratio(element_l1.lower(), matching_element_l2.lower()) / 100 + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": similarity_percentage, + "matching_decision": "fuzzy" + }) else: non_matching_elements_l1.append(element_l1.strip("'")) - + non_matching_elements_l2 = [ element_l2.strip("'") for element_l2 in l2 if element_l2.strip("'") not in matching_elements_l2 ] - - similar_elements = [] - for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) / 100 - matching_condition = "Fuzzy" - similar_elements.append({ - "element_name_l1": element_l1, - "element_name_l2": element_l2, - "similarity_percentage": similarity_percentage, - "matching_condition": matching_condition # Adding matching condition to the result - }) - - result = {"similar_elements": similar_elements} + result = { + "similar_elements": similar_elements + } return result + #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: try: @@ -400,13 +397,14 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo 'value': match['value'], 'similarity_percentage': element['similarity_percentage'], 'confidence': confidence, - 'matching_condition': element["matching_condition"], + 'matching_decision': element["matching_decision"], 'isCustom': is_custom }) processed_labels.add(element['element_name_l1']) else: print(f"No matched data found for {element['element_name_l1']}") + for data in response_data: if data['label'] not in processed_labels: confidence = get_confidence_level(0, score_collection) @@ -419,7 +417,7 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo 'value': data['value'], 'similarity_percentage': 0, 'confidence': confidence, - 'matching_condition': "", + 'matching_decision': "", 'isCustom': False # Explicitly false since there's no l2 match }) @@ -451,6 +449,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str print(f"No match found for '{field}'") return field, None # Return original field if no match is found + #------- works on nested conditions also def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: mapped_nested_data = {} @@ -569,6 +568,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + #print("result: ", result) request_id = generate_request_id() @@ -606,7 +606,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): "value": "$data.value", "similarity_percentage": "$data.similarity_percentage", "confidence": "$data.confidence", - "matching_condition": "$data.matching_condition", + "matching_decision": "$data.matching_decision", "isCustom": "$data.isCustom" }} ]) @@ -625,7 +625,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): "value": doc["value"], "similarity_percentage": doc["similarity_percentage"], "confidence": doc["confidence"], - "matching_condition": doc["matching_condition"], + "matching_decision": doc["matching_decision"], "isCustom": doc["isCustom"] } json_serializable_response.append(json_serializable_doc) From 62593b13baaee88cf3ba73dc65ed8ff5deaf9f5e Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Mar 2024 14:25:26 +0530 Subject: [PATCH 49/91] logic updated for correctly matched and return the similarity score --- dev.py | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/dev.py b/dev.py index a572a08..f473747 100644 --- a/dev.py +++ b/dev.py @@ -689,15 +689,47 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + #-------------------Api fpr storing the admin final policymap for training purpose----------- -@app.post("/generativeaisrvc/store_data") -async def store_data(payload: dict, tenant: str = Header(None)): - try: - policymap_colection = stored_admin_policymap(tenant) - policymap_colection.insert_one(payload) - return {"message": "Data saved successfully"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) +# @app.post("/generativeaisrvc/store_data") +# async def store_data(payload: dict, tenant: str = Header(None)): +# try: +# request_Id = payload.get("request_id") +# policymap_colection = stored_admin_policymap(tenant) +# policymap_colection.insert_one(payload) + +# logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") + +# # query AI suggestion collection +# subset_collection = stored_policy_mapped(tenant) +# doc1 = subset_collection.find_one(request_Id) + +# # query admin collection +# doc2 = policymap_colection.find_one(request_Id) + +# #query global collection +# synonyms_collection = get_master_collection("amayaSynonymsMaster") + +# if doc1 and doc2: +# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): +# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + +# #add logic if synonyms not present add new synonyms to respective key +# # Fetch and update the global collection document +# l2_matched = policy1.get("l2_matched") +# global_doc = synonyms_collection.find_one({"_id": l2_matched}) +# if global_doc: +# new_score = global_doc.get("score", 1) - 0.2 +# synonyms_collection.update_one({"_id": l2_matched}, {"$set": {"score": new_score}}) +# print(f"Updated score for {l2_matched} to {new_score}") +# else: +# print("Documents with the given request_id not found in one or both collections.") + + +# #compare fields and make calculation to update the in global collection +# return {"message": "Data saved successfully"} +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": From b5e1a627f566185152082e8f67039411ccc89313 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Mar 2024 14:58:01 +0530 Subject: [PATCH 50/91] file updated with logic for synonyms --- dev.py | 23 + policy_mapping.py | 369 +++++++--- test.py | 1783 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 2056 insertions(+), 119 deletions(-) create mode 100644 test.py diff --git a/dev.py b/dev.py index f473747..834174c 100644 --- a/dev.py +++ b/dev.py @@ -726,6 +726,29 @@ async def map_fields_to_policy(payload: Dict[str, Any]): # print("Documents with the given request_id not found in one or both collections.") + +# if doc1 and doc2: +# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): +# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): +# # Fetching attributeName from doc1 +# attribute_name = doc1.get("attributeName") + +# # Fetching l2_matched from doc1 +# l2_matched = policy1.get("l2_matched") + +# # Finding the attribute in the global collection +# global_doc = synonyms_collection.find_one({"synonyms.{}".format(attribute_name): {"$exists": True}}) + +# if global_doc: +# new_score = global_doc.get("synonyms", {}).get(attribute_name, {}).get("score", 1) - 0.2 + +# # Updating the global collection with the new score +# synonyms_collection.update_one({"_id": global_doc["_id"], "synonyms.{}".format(attribute_name): {"$exists": True}}, +# {"$set": {"synonyms.{}.score".format(attribute_name): new_score}}) + +# print(f"Updated score for {attribute_name} to {new_score}") + + # #compare fields and make calculation to update the in global collection # return {"message": "Data saved successfully"} # except Exception as e: diff --git a/policy_mapping.py b/policy_mapping.py index 837a3e1..834174c 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -9,7 +9,7 @@ from typing import List, Union import uvicorn from typing import List, Dict, Union -from database.connection import get_collection +from database.connection import get_collection, get_master_collection from dateutil.parser import parse from datetime import datetime from datetime import date @@ -19,6 +19,7 @@ from fuzzywuzzy import process import uvicorn import uuid +from typing import Set app = FastAPI() @@ -27,6 +28,7 @@ format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) + def ResponseModel(data, message, code=200, error_code=None): return { "data": data, @@ -38,18 +40,28 @@ def ResponseModel(data, message, code=200, error_code=None): def ErrorResponseModel(error, code, message): return {"error": error, "code": code, "message": message} +#--------- stored the payload as input---------- def stored_input(tenant: str): return get_collection(tenant, "schema_maker_input") +#--------------stored policymap for all users---------------- def stored_response(tenant: str): return get_collection(tenant, "schema_maker_final_output") +#------------subset policy map response-------------- def stored_policy_mapped(tenant: str): return get_collection(tenant, "schema_maker_policyMap") + +#------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): return get_collection(tenant, "schema_maker_final_policyMap") +#----------custom attributes for appending in cymmetri list----- +def retrieve_custom_attributes(tenant: str): + return get_collection(tenant, "custome_attribute_master") + +#----------- score for confidence level def stored_score(tenant: str, appId: str): score_collection = get_collection(tenant, "schema_maker_score") @@ -60,9 +72,9 @@ def stored_score(tenant: str, appId: str): # if not index_exists: # score_collection.create_index("appId", unique=True) confidence_levels = { - "HIGH": [70, 100], - "LOW": [0, 30], - "MEDIUM": [31, 69] + "HIGH": [0.7, 1], + "LOW": [0, 0.3], + "MEDIUM": [0.31, 0.69] } # Update or insert a single document for the given appId with confidence levels as fields score_collection.update_one( @@ -74,6 +86,7 @@ def stored_score(tenant: str, appId: str): return score_collection +#------ for preprocess purpose----------- def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] @@ -83,6 +96,32 @@ def generate_request_id(): id = uuid.uuid1() return id.hex +#--------------for adding custome attributes in cymmetri field list------------ +def add_custom_attributes_to_list(l2, l2_datatypes, tenant): + + attribute_collection = retrieve_custom_attributes(tenant) + + #query_result = attribute_collection.find({"attributeType": "USER", "status": True}) + query_result = attribute_collection.find({"attributeType": "USER", "status": True}) + + + logging.debug("query executed successfully") + + custom_attributes = [] # This will track the names of custom attributes added + + for result in query_result: + custom_attribute_name = result['name'] + custom_attribute_type = result['provAttributeType'] + + if custom_attribute_name not in l2: + l2.append(custom_attribute_name) + custom_attributes.append(custom_attribute_name) # Add to custom_attributes set if it's new + + l2_datatypes[custom_attribute_name] = custom_attribute_type + + return l2, l2_datatypes, custom_attributes + + #-----------------------------extracting the user object from response----------------- def extract_user_data(response): try: @@ -189,103 +228,116 @@ def get_data_type(value): #-------------------fuzzy logic matching function---------------------- - -# def compare_lists_with_fuzzy(l1, l2, threshold=50): -# logging.debug(f"comparing logic for list1 and list2") - -# matching_elements_l1 = [] -# matching_elements_l2 = [] -# non_matching_elements_l1 = [] -# non_matching_elements_l2 = [] - -# for element_l1 in l1: -# max_similarity = 0 -# matching_element_l2 = '' - -# for element_l2 in l2: -# el1 = str(element_l1).lower() -# el2 = str(element_l2).lower() -# similarity = fuzz.ratio(el1, el2) -# if similarity > max_similarity and similarity >= threshold: -# max_similarity = similarity -# matching_element_l2 = element_l2 - -# if matching_element_l2: -# matching_elements_l1.append(element_l1.strip("'")) -# matching_elements_l2.append(matching_element_l2.strip("'")) -# else: -# non_matching_elements_l1.append(element_l1.strip("'")) - -# non_matching_elements_l2 = [ -# element_l2.strip("'") -# for element_l2 in l2 -# if element_l2.strip("'") not in matching_elements_l2 -# ] - -# similar_elements = [] -# for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): -# similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) -# similar_elements.append({ -# "element_name_l1": element_l1, -# "element_name_l2": element_l2, -# "similarity_percentage": similarity_percentage -# }) - -# result = {"similar_elements": similar_elements} -# return result - - -def compare_lists_with_fuzzy(l1, l2, threshold=50): - logging.debug(f"comparing logic for list1 and list2") - +def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): matching_elements_l1 = [] matching_elements_l2 = [] non_matching_elements_l1 = [] non_matching_elements_l2 = [] similar_elements = [] - + for element_l1 in l1: max_similarity = 0 matching_element_l2 = '' - + is_synonym_match = False # Flag to track if a synonym match is found + + # Check similarity with original list (l2) for element_l2 in l2: el1 = str(element_l1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio(el1, el2)/ 100 + similarity = fuzz.ratio(el1, el2) if similarity > max_similarity and similarity >= threshold: max_similarity = similarity matching_element_l2 = element_l2 - similar_elements.append({ - "element_name_l1": el1, - "element_name_l2": el2, - "similarity_percentage": similarity - }) - + + if not matching_element_l2: + synonyms_doc = synonyms_collection.find_one() + if synonyms_doc: + threshold_synonyms = threshold / 100 + data = synonyms_collection.aggregate([ + { + "$project": { + "synonymsArray": {"$objectToArray": "$synonyms"} + } + }, + {"$unwind": "$synonymsArray"}, + { + "$match": { + "synonymsArray.v": { + "$elemMatch": { + "synonym": el1, + "score": {"$gt": threshold_synonyms} + } + } + } + }, + { + "$project": { + "_id": 0, + "key": "$synonymsArray.k", + "synonyms": { + "$filter": { + "input": "$synonymsArray.v", + "as": "syn", + "cond": { + "$and": [ + {"$eq": ["$$syn.synonym", el1]}, + {"$gt": ["$$syn.score", threshold_synonyms]} + ] + } + } + } + } + }, + {"$limit": 1} + ]) + if data: + for document in data: + matching_element_l2 = document.get('key', None) + synonyms = document.get('synonyms', []) + max_similarity = None + if synonyms: + max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) + is_synonym_match = True + break + + else: + matching_element_l2 = None + max_similarity = None + if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) matching_elements_l2.append(matching_element_l2.strip("'")) + if is_synonym_match: + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": max_similarity, + "matching_decision": "synonyms" + }) + else: + similarity_percentage = fuzz.ratio(element_l1.lower(), matching_element_l2.lower()) / 100 + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": similarity_percentage, + "matching_decision": "fuzzy" + }) else: non_matching_elements_l1.append(element_l1.strip("'")) - - # non_matching_elements_l2 = [ - # element_l2.strip("'") - # for element_l2 in l2 - # if element_l2.strip("'") not in matching_elements_l2 - # ] - - # similar_elements = [] - # for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - # similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) - # similar_elements.append({ - # "element_name_l1": element_l1, - # "element_name_l2": element_l2, - # "similarity_percentage": similarity_percentage - # }) - - result = {"similar_elements": similar_elements} + + non_matching_elements_l2 = [ + element_l2.strip("'") + for element_l2 in l2 + if element_l2.strip("'") not in matching_elements_l2 + ] + + result = { + "similar_elements": similar_elements + } return result + #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: try: @@ -307,7 +359,7 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: elif similarity_score >= score_doc['LOW'][0] and similarity_score <= score_doc['LOW'][1]: return "LOW" else: - return "Unknown" + return "Unknown" # Should not happen if the schema is properly defined else: return "Unknown" # No matching range found, return Unknown except Exception as e: @@ -315,21 +367,27 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: #----------------------generates final response--------------- -def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], response_data: List[Dict[str, str]], l2_datatypes: Dict[str, str], score_collection) -> List[Dict[str, Union[str, int, float]]]: - logging.debug(f"Beautifying the response for saving into the collection") +def generate_final_response(similar_elements: List[Dict[str, Union[str, int, float]]], + response_data: List[Dict[str, str]], + l2_datatypes: Dict[str, str], + score_collection, + custom_attributes: List[str]) -> List[Dict[str, Union[str, int, float]]]: + logging.debug("Beautifying the response for saving into the collection") final_response = [] + processed_labels = set() - # Create a dictionary for easy lookup of response_data based on labels - response_lookup = {data['label']: data for data in response_data} for element in similar_elements: matched_data = [data for data in response_data if data['label'] == element['element_name_l1']] if matched_data: for match in matched_data: l2_datatype = l2_datatypes.get(element['element_name_l2'], None) - # Query the schema_maker_score collection to get the confidence level confidence = get_confidence_level(element['similarity_percentage'], score_collection) + + # Determine if the attribute is custom based on the existence in the custom attributes set + is_custom = element['element_name_l2'] in custom_attributes + final_response.append({ 'jsonPath': match['jsonpath'], 'attributeName': element['element_name_l1'], @@ -338,32 +396,35 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int]]], 'l2_datatype': l2_datatype, 'value': match['value'], 'similarity_percentage': element['similarity_percentage'], - 'confidence': confidence # Include confidence level + 'confidence': confidence, + 'matching_decision': element["matching_decision"], + 'isCustom': is_custom }) - processed_labels.add(element['element_name_l1']) # Track processed labels + processed_labels.add(element['element_name_l1']) else: print(f"No matched data found for {element['element_name_l1']}") - # Handle unmatched elements from l1 + for data in response_data: if data['label'] not in processed_labels: - # Query the schema_maker_score collection to get the confidence level - confidence = get_confidence_level(0,score_collection) # Default to 0 for unmatched elements + confidence = get_confidence_level(0, score_collection) final_response.append({ 'jsonPath': data['jsonpath'], 'attributeName': data['label'], 'l1_datatype': data['datatype'], - 'l2_matched': '', # No match from l2 + 'l2_matched': '', 'l2_datatype': '', - 'value': data['value'], # Use value from response_data - 'similarity_percentage': 0, # Default to 0 for unmatched elements - 'confidence': confidence # Include confidence level + 'value': data['value'], + 'similarity_percentage': 0, + 'confidence': confidence, + 'matching_decision': "", + 'isCustom': False # Explicitly false since there's no l2 match }) return final_response - +#--------------- for mapping the body in body populating api------------------ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: matched = False # Perform case-insensitive exact match @@ -389,6 +450,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str return field, None # Return original field if no match is found +#------- works on nested conditions also def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: mapped_nested_data = {} for field, value in nested_field.items(): @@ -405,19 +467,20 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data - -## Read header as tenant #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') -async def get_mapped(data: dict, tenant: str = Header(None)): +async def get_mapped(data: dict, tenant: str = Header(...)): print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: - + + synonyms_collection = get_master_collection("amayaSynonymsMaster") input_collection = stored_input(tenant) output_collection = stored_response(tenant) subset_collection = stored_policy_mapped(tenant) + custom_attributes = [] + # Store the received response directly into the input collection #input_collection.insert_one(data) @@ -471,11 +534,11 @@ async def get_mapped(data: dict, tenant: str = Header(None)): 'email': 'STRING', 'end_date': 'DATE', 'firstName': 'STRING', - 'login': 'INTEGER', + 'login': 'STRING', 'lastName': 'STRING', 'userType': 'STRING', 'end_date': 'DATE', - 'login': 'INTEGER', + 'login': 'STRING ', 'userType': 'STRING', 'dateOfBirth': 'DATE', 'endDate': 'DATE', @@ -487,24 +550,32 @@ async def get_mapped(data: dict, tenant: str = Header(None)): 'landline': 'STRING' } + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + + print("list2: ",l2) + + #print("custom_attributes: ", custom_attributes) + if isinstance(l2, str): l2_list = convert_string_to_list(l2) else: l2_list = l2 threshold = 60 - - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - print("result: ",result) - appId = data.get("appId") - request_id = generate_request_id() + #--- Inserting the synonyms_dict for appId-------- + #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + #print("result: ", result) + request_id = generate_request_id() + score_collection = stored_score(tenant, appId) - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection) + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) + #print("final response: ",final_response) final_response_dict = {"final_response": final_response} # Assuming 'appId' is present in the received response @@ -534,7 +605,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l2_datatype": "$data.l2_datatype", "value": "$data.value", "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence" + "confidence": "$data.confidence", + "matching_decision": "$data.matching_decision", + "isCustom": "$data.isCustom" }} ]) @@ -551,7 +624,9 @@ async def get_mapped(data: dict, tenant: str = Header(None)): "l2_datatype": doc["l2_datatype"], "value": doc["value"], "similarity_percentage": doc["similarity_percentage"], - "confidence": doc["confidence"] + "confidence": doc["confidence"], + "matching_decision": doc["matching_decision"], + "isCustom": doc["isCustom"] } json_serializable_response.append(json_serializable_doc) @@ -559,7 +634,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): aggregated_data = { "appId": appId, "request_id": request_id, - "subset_data": subset_response_data + "policyMapList": subset_response_data } subset_collection.insert_one(aggregated_data) @@ -580,7 +655,7 @@ async def get_mapped(data: dict, tenant: str = Header(None)): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - +#------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy/") async def map_fields_to_policy(payload: Dict[str, Any]): try: @@ -614,14 +689,70 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") -@app.post("/generativeaisrvc/store_data") -async def store_data(payload: dict, tenant: str = Header(None)): - try: - policymap_colection = stored_admin_policymap(tenant) - policymap_colection.insert_one(payload) - return {"message": "Data saved successfully"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + +#-------------------Api fpr storing the admin final policymap for training purpose----------- +# @app.post("/generativeaisrvc/store_data") +# async def store_data(payload: dict, tenant: str = Header(None)): +# try: +# request_Id = payload.get("request_id") +# policymap_colection = stored_admin_policymap(tenant) +# policymap_colection.insert_one(payload) + +# logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") + +# # query AI suggestion collection +# subset_collection = stored_policy_mapped(tenant) +# doc1 = subset_collection.find_one(request_Id) + +# # query admin collection +# doc2 = policymap_colection.find_one(request_Id) + +# #query global collection +# synonyms_collection = get_master_collection("amayaSynonymsMaster") + +# if doc1 and doc2: +# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): +# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + +# #add logic if synonyms not present add new synonyms to respective key +# # Fetch and update the global collection document +# l2_matched = policy1.get("l2_matched") +# global_doc = synonyms_collection.find_one({"_id": l2_matched}) +# if global_doc: +# new_score = global_doc.get("score", 1) - 0.2 +# synonyms_collection.update_one({"_id": l2_matched}, {"$set": {"score": new_score}}) +# print(f"Updated score for {l2_matched} to {new_score}") +# else: +# print("Documents with the given request_id not found in one or both collections.") + + + +# if doc1 and doc2: +# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): +# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): +# # Fetching attributeName from doc1 +# attribute_name = doc1.get("attributeName") + +# # Fetching l2_matched from doc1 +# l2_matched = policy1.get("l2_matched") + +# # Finding the attribute in the global collection +# global_doc = synonyms_collection.find_one({"synonyms.{}".format(attribute_name): {"$exists": True}}) + +# if global_doc: +# new_score = global_doc.get("synonyms", {}).get(attribute_name, {}).get("score", 1) - 0.2 + +# # Updating the global collection with the new score +# synonyms_collection.update_one({"_id": global_doc["_id"], "synonyms.{}".format(attribute_name): {"$exists": True}}, +# {"$set": {"synonyms.{}.score".format(attribute_name): new_score}}) + +# print(f"Updated score for {attribute_name} to {new_score}") + + +# #compare fields and make calculation to update the in global collection +# return {"message": "Data saved successfully"} +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": diff --git a/test.py b/test.py new file mode 100644 index 0000000..bbb723d --- /dev/null +++ b/test.py @@ -0,0 +1,1783 @@ +{ + "_id": { + "$oid": "65fd5b8f823f1e33f9d29f2e" + }, + "synonyms": { + "department": [ + { + "synonym": "division", + "score": 1 + }, + { + "synonym": "section", + "score": 1 + }, + { + "synonym": "unit", + "score": 1 + }, + { + "synonym": "branch", + "score": 1 + }, + { + "synonym": "sector", + "score": 1 + }, + { + "synonym": "departmentalunit", + "score": 1 + }, + { + "synonym": "team", + "score": 1 + }, + { + "synonym": "segment", + "score": 1 + }, + { + "synonym": "group", + "score": 1 + }, + { + "synonym": "subdivision", + "score": 1 + }, + { + "synonym": "area", + "score": 1 + }, + { + "synonym": "office", + "score": 1 + }, + { + "synonym": "bureau", + "score": 1 + }, + { + "synonym": "wing", + "score": 1 + }, + { + "synonym": "part", + "score": 1 + }, + { + "synonym": "component", + "score": 1 + }, + { + "synonym": "sphere", + "score": 1 + }, + { + "synonym": "category", + "score": 1 + }, + { + "synonym": "subunit", + "score": 1 + }, + { + "synonym": "subsection", + "score": 1 + } + ], + "employeeId": [ + { + "synonym": "staffID", + "score": 1 + }, + { + "synonym": "workerID", + "score": 1 + }, + { + "synonym": "employeenumber", + "score": 1 + }, + { + "synonym": "identificationnumber", + "score": 1 + }, + { + "synonym": "personnelcode", + "score": 1 + }, + { + "synonym": "staffcode", + "score": 1 + }, + { + "synonym": "workercode", + "score": 1 + }, + { + "synonym": "employeereference", + "score": 1 + }, + { + "synonym": "staffreference", + "score": 1 + }, + { + "synonym": "personnelnumber", + "score": 1 + }, + { + "synonym": "employmentID", + "score": 1 + }, + { + "synonym": "jobID", + "score": 1 + }, + { + "synonym": "workID", + "score": 1 + }, + { + "synonym": "staffidentification", + "score": 1 + }, + { + "synonym": "staffreferencenumber", + "score": 1 + }, + { + "synonym": "employeeidentifier", + "score": 1 + }, + { + "synonym": "workeridentifier", + "score": 1 + }, + { + "synonym": "employmentnumber", + "score": 1 + }, + { + "synonym": "personnelID", + "score": 1 + }, + { + "synonym": "personnelidentifier", + "score": 1 + } + ], + "designation": [ + { + "synonym": "title", + "score": 1 + }, + { + "synonym": "position", + "score": 1 + }, + { + "synonym": "role", + "score": 1 + }, + { + "synonym": "rank", + "score": 1 + }, + { + "synonym": "jobtitle", + "score": 1 + }, + { + "synonym": "function", + "score": 1 + }, + { + "synonym": "appointment", + "score": 1 + }, + { + "synonym": "capacity", + "score": 1 + }, + { + "synonym": "post", + "score": 1 + }, + { + "synonym": "task", + "score": 1 + }, + { + "synonym": "duty", + "score": 1 + }, + { + "synonym": "responsibility", + "score": 1 + }, + { + "synonym": "occupation", + "score": 1 + }, + { + "synonym": "roletitle", + "score": 1 + }, + { + "synonym": "positiontitle", + "score": 1 + }, + { + "synonym": "worktitle", + "score": 1 + }, + { + "synonym": "jobrole", + "score": 1 + }, + { + "synonym": "jobposition", + "score": 1 + }, + { + "synonym": "jobdesignation", + "score": 1 + }, + { + "synonym": "functiontitle", + "score": 1 + } + ], + "appUpdatedDate": [ + { + "synonym": "applicationupdatedate", + "score": 1 + }, + { + "synonym": "softwareupdatedate", + "score": 1 + }, + { + "synonym": "appmodificationdate", + "score": 1 + }, + { + "synonym": "updatetimestamp", + "score": 1 + }, + { + "synonym": "softwarerevisiondate", + "score": 1 + }, + { + "synonym": "appupgradedate", + "score": 1 + }, + { + "synonym": "programupdatedate", + "score": 1 + }, + { + "synonym": "softwarerefreshdate", + "score": 1 + }, + { + "synonym": "appenhancementdate", + "score": 1 + }, + { + "synonym": "modificationtimestamp", + "score": 1 + }, + { + "synonym": "upgradetimestamp", + "score": 1 + }, + { + "synonym": "revisiontimestamp", + "score": 1 + }, + { + "synonym": "refreshtimestamp", + "score": 1 + }, + { + "synonym": "enhancementtimestamp", + "score": 1 + }, + { + "synonym": "softwarechangedate", + "score": 1 + }, + { + "synonym": "appchangedate", + "score": 1 + }, + { + "synonym": "changetimestamp", + "score": 1 + }, + { + "synonym": "updatetime", + "score": 1 + }, + { + "synonym": "modificationdate", + "score": 1 + }, + { + "synonym": "revisiondate", + "score": 1 + } + ], + "displayName": [ + { + "synonym": "namedisplayed", + "score": 1 + }, + { + "synonym": "visiblename", + "score": 1 + }, + { + "synonym": "publicname", + "score": 1 + }, + { + "synonym": "nameshown", + "score": 1 + }, + { + "synonym": "presentationname", + "score": 1 + }, + { + "synonym": "exhibitedname", + "score": 1 + }, + { + "synonym": "publiclyshownname", + "score": 1 + }, + { + "synonym": "visibletitle", + "score": 1 + }, + { + "synonym": "exhibitedtitle", + "score": 1 + }, + { + "synonym": "publiclyshowntitle", + "score": 1 + }, + { + "synonym": "presentationtitle", + "score": 1 + }, + { + "synonym": "showntitle", + "score": 1 + }, + { + "synonym": "displayedtitle", + "score": 1 + }, + { + "synonym": "visibledesignation", + "score": 1 + }, + { + "synonym": "exhibiteddesignation", + "score": 1 + }, + { + "synonym": "publiclyshowndesignation", + "score": 1 + }, + { + "synonym": "presentationdesignation", + "score": 1 + }, + { + "synonym": "showndesignation", + "score": 1 + }, + { + "synonym": "displayeddesignation", + "score": 1 + }, + { + "synonym": "displayedidentity", + "score": 1 + } + ], + "mobile": [ + { + "synonym": "cellular", + "score": 1 + }, + { + "synonym": "cellphone", + "score": 1 + }, + { + "synonym": "mobilephone", + "score": 1 + }, + { + "synonym": "wirelessphone", + "score": 1 + }, + { + "synonym": "portablephone", + "score": 1 + }, + { + "synonym": "handset", + "score": 1 + }, + { + "synonym": "smartphone", + "score": 1 + }, + { + "synonym": "cell", + "score": 1 + }, + { + "synonym": "mobiledevice", + "score": 1 + }, + { + "synonym": "mobilecellular", + "score": 1 + }, + { + "synonym": "portabledevice", + "score": 1 + }, + { + "synonym": "wirelessdevice", + "score": 1 + }, + { + "synonym": "cellulardevice", + "score": 1 + }, + { + "synonym": "handhelddevice", + "score": 1 + }, + { + "synonym": "cellularphone", + "score": 1 + }, + { + "synonym": "cellulartelephone", + "score": 1 + }, + { + "synonym": "mobileunit", + "score": 1 + }, + { + "synonym": "wirelessunit", + "score": 1 + } + ], + "country": [ + { + "synonym": "nation", + "score": 1 + }, + { + "synonym": "state", + "score": 1 + }, + { + "synonym": "territory", + "score": 1 + }, + { + "synonym": "land", + "score": 1 + }, + { + "synonym": "countrystate", + "score": 1 + }, + { + "synonym": "nationstate", + "score": 1 + }, + { + "synonym": "realm", + "score": 1 + }, + { + "synonym": "region", + "score": 1 + }, + { + "synonym": "commonwealth", + "score": 1 + }, + { + "synonym": "province", + "score": 1 + }, + { + "synonym": "domain", + "score": 1 + }, + { + "synonym": "sovereignstate", + "score": 1 + }, + { + "synonym": "nationalterritory", + "score": 1 + }, + { + "synonym": "nationterritory", + "score": 1 + }, + { + "synonym": "countryterritory", + "score": 1 + }, + { + "synonym": "homeland", + "score": 1 + }, + { + "synonym": "fatherland", + "score": 1 + }, + { + "synonym": "motherland", + "score": 1 + }, + { + "synonym": "nativeland", + "score": 1 + }, + { + "synonym": "soil", + "score": 1 + } + ], + "city": [ + { + "synonym": "metropolis", + "score": 1 + }, + { + "synonym": "urbancenter", + "score": 1 + }, + { + "synonym": "town", + "score": 1 + }, + { + "synonym": "municipality", + "score": 1 + }, + { + "synonym": "cityscape", + "score": 1 + }, + { + "synonym": "borough", + "score": 1 + }, + { + "synonym": "locality", + "score": 1 + }, + { + "synonym": "urbanarea", + "score": 1 + }, + { + "synonym": "downtown", + "score": 1 + }, + { + "synonym": "community", + "score": 1 + }, + { + "synonym": "village", + "score": 1 + }, + { + "synonym": "conurbation", + "score": 1 + }, + { + "synonym": "township", + "score": 1 + }, + { + "synonym": "megalopolis", + "score": 1 + }, + { + "synonym": "cosmopolis", + "score": 1 + }, + { + "synonym": "megalopolitanarea", + "score": 1 + }, + { + "synonym": "metropolitanarea", + "score": 1 + }, + { + "synonym": "megalopolitanregion", + "score": 1 + }, + { + "synonym": "citycenter", + "score": 1 + }, + { + "synonym": "citydistrict", + "score": 1 + } + ], + "email": [ + { + "synonym": "electronicmail", + "score": 1 + }, + { + "synonym": "emailaddress", + "score": 1 + }, + { + "synonym": "e-message", + "score": 1 + }, + { + "synonym": "emailcorrespondence", + "score": 1 + }, + { + "synonym": "digitalmail", + "score": 1 + }, + { + "synonym": "e-mail", + "score": 1 + }, + { + "synonym": "internetmail", + "score": 1 + }, + { + "synonym": "onlinemail", + "score": 1 + }, + { + "synonym": "electroniccorrespondence", + "score": 1 + }, + { + "synonym": "cybermail", + "score": 1 + }, + { + "synonym": "virtualmail", + "score": 1 + }, + { + "synonym": "webmail", + "score": 1 + }, + { + "synonym": "internetmessage", + "score": 1 + }, + { + "synonym": "e-post", + "score": 1 + }, + { + "synonym": "e-letter", + "score": 1 + }, + { + "synonym": "electronicmessage", + "score": 1 + }, + { + "synonym": "e-communique", + "score": 1 + }, + { + "synonym": "digitalmessage", + "score": 1 + }, + { + "synonym": "onlinemessage", + "score": 1 + }, + { + "synonym": "webmessage", + "score": 1 + } + ], + "end_date": [ + { + "synonym": "enddate", + "score": 1 + }, + { + "synonym": "terminationdate", + "score": 1 + }, + { + "synonym": "conclusiondate", + "score": 1 + }, + { + "synonym": "finishdate", + "score": 1 + }, + { + "synonym": "completiondate", + "score": 1 + }, + { + "synonym": "closingdate", + "score": 1 + }, + { + "synonym": "expirationdate", + "score": 1 + }, + { + "synonym": "finaldate", + "score": 1 + }, + { + "synonym": "culminationdate", + "score": 1 + }, + { + "synonym": "endingdate", + "score": 1 + }, + { + "synonym": "expirydate", + "score": 1 + }, + { + "synonym": "concludingdate", + "score": 1 + }, + { + "synonym": "ceasingdate", + "score": 1 + }, + { + "synonym": "lastdate", + "score": 1 + }, + { + "synonym": "terminationtime", + "score": 1 + }, + { + "synonym": "conclusiontime", + "score": 1 + }, + { + "synonym": "finishtime", + "score": 1 + }, + { + "synonym": "completiontime", + "score": 1 + }, + { + "synonym": "closingtime", + "score": 1 + }, + { + "synonym": "expirationtime", + "score": 1 + } + ], + "firstName": [ + { + "synonym": "givenname", + "score": 1 + }, + { + "synonym": "forename", + "score": 1 + }, + { + "synonym": "firstname", + "score": 1 + }, + { + "synonym": "Christianname", + "score": 1 + }, + { + "synonym": "personalname", + "score": 1 + }, + { + "synonym": "individualname", + "score": 1 + }, + { + "synonym": "giventitle", + "score": 1 + }, + { + "synonym": "initialname", + "score": 1 + }, + { + "synonym": "name", + "score": 1 + }, + { + "synonym": "givenappellation", + "score": 1 + }, + { + "synonym": "appellation", + "score": 1 + }, + { + "synonym": "nametag", + "score": 1 + }, + { + "synonym": "namelabel", + "score": 1 + }, + { + "synonym": "givendesignation", + "score": 1 + }, + { + "synonym": "givenidentity", + "score": 1 + }, + { + "synonym": "title", + "score": 1 + }, + { + "synonym": "handle", + "score": 1 + }, + { + "synonym": "moniker", + "score": 1 + }, + { + "synonym": "nickname", + "score": 1 + }, + { + "synonym": "nomenclature", + "score": 1 + } + ], + "login": [ + { + "synonym": "userprincipalname", + "score": 1 + }, + { + "synonym": "sign-in", + "score": 1 + }, + { + "synonym": "logon", + "score": 1 + }, + { + "synonym": "logincredentials", + "score": 1 + }, + { + "synonym": "access", + "score": 1 + }, + { + "synonym": "accesscode", + "score": 1 + }, + { + "synonym": "username", + "score": 1 + }, + { + "synonym": "userID", + "score": 1 + }, + { + "synonym": "loginID", + "score": 1 + }, + { + "synonym": "logonID", + "score": 1 + }, + { + "synonym": "sign-in details", + "score": 1 + }, + { + "synonym": "accessdetails", + "score": 1 + }, + { + "synonym": "accessinformation", + "score": 1 + }, + { + "synonym": "sign-ininformation", + "score": 1 + }, + { + "synonym": "logoninformation", + "score": 1 + }, + { + "synonym": "credentials", + "score": 1 + }, + { + "synonym": "authentication", + "score": 1 + }, + { + "synonym": "loginname", + "score": 1 + }, + { + "synonym": "sign-inname", + "score": 1 + }, + { + "synonym": "logonname", + "score": 1 + }, + { + "synonym": "accessname", + "score": 1 + } + ], + "lastName": [ + { + "synonym": "familyname", + "score": 1 + }, + { + "synonym": "surname", + "score": 1 + }, + { + "synonym": "lastname", + "score": 1 + }, + { + "synonym": "secondname", + "score": 1 + }, + { + "synonym": "patronymic", + "score": 1 + }, + { + "synonym": "matronymic", + "score": 1 + }, + { + "synonym": "sirename", + "score": 1 + }, + { + "synonym": "maidenname", + "score": 1 + }, + { + "synonym": "maidensurname", + "score": 1 + }, + { + "synonym": "parentalname", + "score": 1 + }, + { + "synonym": "parentalsurname", + "score": 1 + }, + { + "synonym": "cognomen", + "score": 1 + }, + { + "synonym": "familytitle", + "score": 1 + }, + { + "synonym": "familyappellation", + "score": 1 + }, + { + "synonym": "familylabel", + "score": 1 + }, + { + "synonym": "familydesignation", + "score": 1 + }, + { + "synonym": "familyidentity", + "score": 1 + }, + { + "synonym": "familyhandle", + "score": 1 + } + ], + "userType": [ + { + "synonym": "roletype", + "score": 1 + }, + { + "synonym": "usercategory", + "score": 1 + }, + { + "synonym": "accounttype", + "score": 1 + }, + { + "synonym": "userrole", + "score": 1 + }, + { + "synonym": "profiletype", + "score": 1 + }, + { + "synonym": "identitytype", + "score": 1 + }, + { + "synonym": "classification", + "score": 1 + }, + { + "synonym": "userclassification", + "score": 1 + }, + { + "synonym": "rolecategory", + "score": 1 + }, + { + "synonym": "userclass", + "score": 1 + }, + { + "synonym": "identityclass", + "score": 1 + }, + { + "synonym": "profileclass", + "score": 1 + }, + { + "synonym": "usergroup", + "score": 1 + }, + { + "synonym": "identitygroup", + "score": 1 + }, + { + "synonym": "profilegroup", + "score": 1 + }, + { + "synonym": "roleclassification", + "score": 1 + }, + { + "synonym": "userroleclassification", + "score": 1 + }, + { + "synonym": "identityroleclassification", + "score": 1 + }, + { + "synonym": "profileroleclassification", + "score": 1 + }, + { + "synonym": "useridentitytype", + "score": 1 + } + ], + "dateOfBirth": [ + { + "synonym": "birthdate", + "score": 1 + }, + { + "synonym": "DOB", + "score": 1 + }, + { + "synonym": "dateofbirth", + "score": 1 + }, + { + "synonym": "natalday", + "score": 1 + }, + { + "synonym": "bornday", + "score": 1 + }, + { + "synonym": "anniversaryofbirth", + "score": 1 + }, + { + "synonym": "nativitydate", + "score": 1 + }, + { + "synonym": "birthday", + "score": 1 + }, + { + "synonym": "borndate", + "score": 1 + }, + { + "synonym": "nataldate", + "score": 1 + }, + { + "synonym": "anniversaryofnativity", + "score": 1 + }, + { + "synonym": "natalanniversary", + "score": 1 + }, + { + "synonym": "bornanniversary", + "score": 1 + }, + { + "synonym": "birthanniversary", + "score": 1 + }, + { + "synonym": "nativityday", + "score": 1 + }, + { + "synonym": "birthdayanniversary", + "score": 1 + } + ], + "endDate": [ + { + "synonym": "enddate", + "score": 1 + }, + { + "synonym": "conclusiondate", + "score": 1 + }, + { + "synonym": "terminationdate", + "score": 1 + }, + { + "synonym": "finishdate", + "score": 1 + }, + { + "synonym": "completiondate", + "score": 1 + }, + { + "synonym": "closingdate", + "score": 1 + }, + { + "synonym": "expirationdate", + "score": 1 + }, + { + "synonym": "finaldate", + "score": 1 + }, + { + "synonym": "culminationdate", + "score": 1 + }, + { + "synonym": "endingdate", + "score": 1 + }, + { + "synonym": "expirydate", + "score": 1 + }, + { + "synonym": "concludingdate", + "score": 1 + }, + { + "synonym": "ceasingdate", + "score": 1 + }, + { + "synonym": "lastdate", + "score": 1 + }, + { + "synonym": "terminationtime", + "score": 1 + }, + { + "synonym": "conclusiontime", + "score": 1 + }, + { + "synonym": "finishtime", + "score": 1 + }, + { + "synonym": "completiontime", + "score": 1 + }, + { + "synonym": "closingtime", + "score": 1 + }, + { + "synonym": "expirationtime", + "score": 1 + } + ], + "startDate": [ + { + "synonym": "startdate", + "score": 1 + }, + { + "synonym": "commencementdate", + "score": 1 + }, + { + "synonym": "beginningdate", + "score": 1 + }, + { + "synonym": "initiationdate", + "score": 1 + }, + { + "synonym": "commencingdate", + "score": 1 + }, + { + "synonym": "onsetdate", + "score": 1 + }, + { + "synonym": "commencementtime", + "score": 1 + }, + { + "synonym": "initiationtime", + "score": 1 + }, + { + "synonym": "starttime", + "score": 1 + }, + { + "synonym": "commencingtime", + "score": 1 + }, + { + "synonym": "onsettime", + "score": 1 + }, + { + "synonym": "commencementpoint", + "score": 1 + }, + { + "synonym": "initiationpoint", + "score": 1 + }, + { + "synonym": "startingpoint", + "score": 1 + }, + { + "synonym": "commencingpoint", + "score": 1 + }, + { + "synonym": "onsetpoint", + "score": 1 + }, + { + "synonym": "launchdate", + "score": 1 + }, + { + "synonym": "kickoffdate", + "score": 1 + }, + { + "synonym": "openingdate", + "score": 1 + }, + { + "synonym": "inaugurationdate", + "score": 1 + } + ], + "password": [ + { + "synonym": "passcode", + "score": 1 + }, + { + "synonym": "accesscode", + "score": 1 + }, + { + "synonym": "securitycode", + "score": 1 + }, + { + "synonym": "logincode", + "score": 1 + }, + { + "synonym": "passphrase", + "score": 1 + }, + { + "synonym": "authenticationcode", + "score": 1 + }, + { + "synonym": "key", + "score": 1 + }, + { + "synonym": "secretkey", + "score": 1 + }, + { + "synonym": "code", + "score": 1 + }, + { + "synonym": "PIN", + "score": 1 + }, + { + "synonym": "loginkey", + "score": 1 + }, + { + "synonym": "accesskey", + "score": 1 + }, + { + "synonym": "passkey", + "score": 1 + }, + { + "synonym": "securitykey", + "score": 1 + }, + { + "synonym": "identificationcode", + "score": 1 + }, + { + "synonym": "authenticationkey", + "score": 1 + }, + { + "synonym": "cipher", + "score": 1 + }, + { + "synonym": "loginpassword", + "score": 1 + }, + { + "synonym": "securitypassword", + "score": 1 + }, + { + "synonym": "accesspassword", + "score": 1 + } + ], + "status": [ + { + "synonym": "condition", + "score": 1 + }, + { + "synonym": "state", + "score": 1 + }, + { + "synonym": "situation", + "score": 1 + }, + { + "synonym": "standing", + "score": 1 + }, + { + "synonym": "position", + "score": 1 + }, + { + "synonym": "circumstance", + "score": 1 + }, + { + "synonym": "statusquo", + "score": 1 + }, + { + "synonym": "mode", + "score": 1 + }, + { + "synonym": "stage", + "score": 1 + }, + { + "synonym": "phase", + "score": 1 + }, + { + "synonym": "stateofaffairs", + "score": 1 + }, + { + "synonym": "positioning", + "score": 1 + }, + { + "synonym": "conditioning", + "score": 1 + }, + { + "synonym": "stateofbeing", + "score": 1 + }, + { + "synonym": "statuscondition", + "score": 1 + }, + { + "synonym": "statusstate", + "score": 1 + }, + { + "synonym": "statussituation", + "score": 1 + }, + { + "synonym": "statusposition", + "score": 1 + }, + { + "synonym": "statuscircumstance", + "score": 1 + }, + { + "synonym": "statusphase", + "score": 1 + } + ], + "profilePicture": [ + { + "synonym": "avatar", + "score": 1 + }, + { + "synonym": "userimage", + "score": 1 + }, + { + "synonym": "displaypicture", + "score": 1 + }, + { + "synonym": "profileimage", + "score": 1 + }, + { + "synonym": "profilephoto", + "score": 1 + }, + { + "synonym": "userphoto", + "score": 1 + }, + { + "synonym": "portrait", + "score": 1 + }, + { + "synonym": "icon", + "score": 1 + }, + { + "synonym": "thumbnail", + "score": 1 + }, + { + "synonym": "representation", + "score": 1 + }, + { + "synonym": "graphic", + "score": 1 + }, + { + "synonym": "digitalimage", + "score": 1 + }, + { + "synonym": "visualrepresentation", + "score": 1 + }, + { + "synonym": "picture", + "score": 1 + }, + { + "synonym": "photo", + "score": 1 + }, + { + "synonym": "displayimage", + "score": 1 + }, + { + "synonym": "profileavatar", + "score": 1 + }, + { + "synonym": "useravatar", + "score": 1 + }, + { + "synonym": "image", + "score": 1 + }, + { + "synonym": "profilerepresentation", + "score": 1 + } + ], + "appUserId": [ + { + "synonym": "applicationuserID", + "score": 1 + }, + { + "synonym": "softwareuseridentifier", + "score": 1 + }, + { + "synonym": "appaccountID", + "score": 1 + }, + { + "synonym": "userID", + "score": 1 + }, + { + "synonym": "accountID", + "score": 1 + }, + { + "synonym": "useridentity", + "score": 1 + }, + { + "synonym": "usercode", + "score": 1 + }, + { + "synonym": "useridentifier", + "score": 1 + }, + { + "synonym": "appidentity", + "score": 1 + }, + { + "synonym": "softwareID", + "score": 1 + }, + { + "synonym": "softwareidentifier", + "score": 1 + }, + { + "synonym": "applicationID", + "score": 1 + }, + { + "synonym": "applicationidentifier", + "score": 1 + }, + { + "synonym": "appcode", + "score": 1 + }, + { + "synonym": "softwarecode", + "score": 1 + }, + { + "synonym": "accountcode", + "score": 1 + }, + { + "synonym": "usernumber", + "score": 1 + }, + { + "synonym": "identitynumber", + "score": 1 + }, + { + "synonym": "IDcode", + "score": 1 + }, + { + "synonym": "IDnumber", + "score": 1 + } + ], + "landline": [ + { + "synonym": "fixedline", + "score": 1 + }, + { + "synonym": "homephone", + "score": 1 + }, + { + "synonym": "landlinephone", + "score": 1 + }, + { + "synonym": "landlinenumber", + "score": 1 + }, + { + "synonym": "homephonenumber", + "score": 1 + }, + { + "synonym": "residencephone", + "score": 1 + }, + { + "synonym": "residenceline", + "score": 1 + }, + { + "synonym": "telephonenumber", + "score": 1 + }, + { + "synonym": "fixedphone", + "score": 1 + }, + { + "synonym": "fixedtelephone", + "score": 1 + }, + { + "synonym": "domesticphone", + "score": 1 + }, + { + "synonym": "domesticline", + "score": 1 + }, + { + "synonym": "domestictelphone", + "score": 1 + }, + { + "synonym": "housephone", + "score": 1 + }, + { + "synonym": "houseline", + "score": 1 + }, + { + "synonym": "housetelephone", + "score": 1 + }, + { + "synonym": "wiredphone", + "score": 1 + }, + { + "synonym": "wiredline", + "score": 1 + }, + { + "synonym": "wiredtelephone", + "score": 1 + }, + { + "synonym": "cordedphone", + "score": 1 + } + ] + } +} \ No newline at end of file From 5bdc15bf3b52f4a8fb774e7e9cbeaa5c0793b9c9 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 26 Mar 2024 16:49:59 +0530 Subject: [PATCH 51/91] logic added for nested key in user object --- dev.py | 23 ++++++++++++----------- policy_mapping.py | 23 ++++++++++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/dev.py b/dev.py index 834174c..8a69b35 100644 --- a/dev.py +++ b/dev.py @@ -42,20 +42,20 @@ def ErrorResponseModel(error, code, message): #--------- stored the payload as input---------- def stored_input(tenant: str): - return get_collection(tenant, "schema_maker_input") + return get_collection(tenant, "amaya_input") #--------------stored policymap for all users---------------- def stored_response(tenant: str): - return get_collection(tenant, "schema_maker_final_output") + return get_collection(tenant, "amaya_final_output") #------------subset policy map response-------------- def stored_policy_mapped(tenant: str): - return get_collection(tenant, "schema_maker_policyMap") + return get_collection(tenant, "amaya_policyMap") #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): - return get_collection(tenant, "schema_maker_final_policyMap") + return get_collection(tenant, "amaya_final_policyMap") #----------custom attributes for appending in cymmetri list----- def retrieve_custom_attributes(tenant: str): @@ -63,7 +63,7 @@ def retrieve_custom_attributes(tenant: str): #----------- score for confidence level def stored_score(tenant: str, appId: str): - score_collection = get_collection(tenant, "schema_maker_score") + score_collection = get_collection(tenant, "amaya_score") # Check if index exists #index_exists = appId in score_collection.index_information() @@ -167,11 +167,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -225,7 +226,7 @@ def get_data_type(value): return distinct_keys_datatypes except Exception as e: raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") - + #-------------------fuzzy logic matching function---------------------- def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): diff --git a/policy_mapping.py b/policy_mapping.py index 834174c..8a69b35 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -42,20 +42,20 @@ def ErrorResponseModel(error, code, message): #--------- stored the payload as input---------- def stored_input(tenant: str): - return get_collection(tenant, "schema_maker_input") + return get_collection(tenant, "amaya_input") #--------------stored policymap for all users---------------- def stored_response(tenant: str): - return get_collection(tenant, "schema_maker_final_output") + return get_collection(tenant, "amaya_final_output") #------------subset policy map response-------------- def stored_policy_mapped(tenant: str): - return get_collection(tenant, "schema_maker_policyMap") + return get_collection(tenant, "amaya_policyMap") #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): - return get_collection(tenant, "schema_maker_final_policyMap") + return get_collection(tenant, "amaya_final_policyMap") #----------custom attributes for appending in cymmetri list----- def retrieve_custom_attributes(tenant: str): @@ -63,7 +63,7 @@ def retrieve_custom_attributes(tenant: str): #----------- score for confidence level def stored_score(tenant: str, appId: str): - score_collection = get_collection(tenant, "schema_maker_score") + score_collection = get_collection(tenant, "amaya_score") # Check if index exists #index_exists = appId in score_collection.index_information() @@ -167,11 +167,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -225,7 +226,7 @@ def get_data_type(value): return distinct_keys_datatypes except Exception as e: raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") - + #-------------------fuzzy logic matching function---------------------- def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): From 85e53d2e481aa16884049b6dda9a57acc066182f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 27 Mar 2024 16:59:05 +0530 Subject: [PATCH 52/91] logic updated for feedback api --- dev.py | 218 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 167 insertions(+), 51 deletions(-) diff --git a/dev.py b/dev.py index 8a69b35..4e6c777 100644 --- a/dev.py +++ b/dev.py @@ -52,7 +52,6 @@ def stored_response(tenant: str): def stored_policy_mapped(tenant: str): return get_collection(tenant, "amaya_policyMap") - #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): return get_collection(tenant, "amaya_final_policyMap") @@ -692,69 +691,186 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- -# @app.post("/generativeaisrvc/store_data") -# async def store_data(payload: dict, tenant: str = Header(None)): -# try: -# request_Id = payload.get("request_id") -# policymap_colection = stored_admin_policymap(tenant) -# policymap_colection.insert_one(payload) +@app.post("/generativeaisrvc/store_data") +async def store_data(payload: dict, tenant: str = Header(None)): + print("tenant: ", tenant) + try: + request_Id = payload.get("request_id") + policymap_collection = stored_admin_policymap(tenant) + policymap_collection.insert_one(payload) -# logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") + logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") -# # query AI suggestion collection -# subset_collection = stored_policy_mapped(tenant) -# doc1 = subset_collection.find_one(request_Id) + # query AI suggestion collection + subset_collection = stored_policy_mapped(tenant) + doc1 = subset_collection.find_one({"request_id":request_Id}) -# # query admin collection -# doc2 = policymap_colection.find_one(request_Id) + # query admin collection + doc2 = policymap_collection.find_one({"request_id":request_Id}) -# #query global collection -# synonyms_collection = get_master_collection("amayaSynonymsMaster") + #query global collection + synonyms_collection = get_master_collection("amayaSynonymsMaster") -# if doc1 and doc2: -# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): -# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + if doc1 and doc2: + # print("doc1: ",doc1) + # print("doc2: ",doc2) + for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): + # print("policy1: ",policy1) + # print("policy2: ",policy2) + + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #Fetching attributeName from doc1 + attribute_name = policy1.get("attributeName").lower() + print("attribute_name: ",attribute_name) + + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + print("l2_matched: ",l2_matched) + + # Finding the attribute in the global collection -# #add logic if synonyms not present add new synonyms to respective key -# # Fetch and update the global collection document -# l2_matched = policy1.get("l2_matched") -# global_doc = synonyms_collection.find_one({"_id": l2_matched}) -# if global_doc: -# new_score = global_doc.get("score", 1) - 0.2 -# synonyms_collection.update_one({"_id": l2_matched}, {"$set": {"score": new_score}}) -# print(f"Updated score for {l2_matched} to {new_score}") -# else: -# print("Documents with the given request_id not found in one or both collections.") + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + global_docs = synonyms_collection.aggregate(pipeline) + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + new_score = synonyms.get(attribute_name, {}).get("score",1) - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) -# if doc1 and doc2: -# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): -# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): -# # Fetching attributeName from doc1 -# attribute_name = doc1.get("attributeName") - -# # Fetching l2_matched from doc1 -# l2_matched = policy1.get("l2_matched") + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + print("No 'synonyms' found in the document.") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + attribute_name = policy1.get("attributeName").lower() + print("attribute_name: ",attribute_name) -# # Finding the attribute in the global collection -# global_doc = synonyms_collection.find_one({"synonyms.{}".format(attribute_name): {"$exists": True}}) + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + print("l2_matched: ",l2_matched) -# if global_doc: -# new_score = global_doc.get("synonyms", {}).get(attribute_name, {}).get("score", 1) - 0.2 - -# # Updating the global collection with the new score -# synonyms_collection.update_one({"_id": global_doc["_id"], "synonyms.{}".format(attribute_name): {"$exists": True}}, -# {"$set": {"synonyms.{}.score".format(attribute_name): new_score}}) - -# print(f"Updated score for {attribute_name} to {new_score}") - - -# #compare fields and make calculation to update the in global collection -# return {"message": "Data saved successfully"} + # Finding the attribute in the global collection + + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + new_score = synonyms.get(attribute_name, {}).get("score",1) - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + print("No 'synonyms' found in the document.") + + else: + print("failed") + + #compare fields and make calculation to update the in global collection + return {"message": "Data saved successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +#------------ API for mapping roles------------- +# @app.post("/generativeaisrvc/map_fields_to_roles/") +# async def map_fields_to_roles(payload: Dict[str, Any]): +# try: +# body = payload.get("body") +# policy_mapping = payload.get("policyMapping") + +# if not body: +# raise HTTPException(status_code=400, detail="body empty") +# elif not policy_mapping: +# raise HTTPException(status_code=400, detail="policy_mapping empty") + +# except HTTPException: +# raise # except Exception as e: -# raise HTTPException(status_code=500, detail=str(e)) +# return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From c3d2db2eb1f607688addbc140fe8ee227a39e7e4 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 27 Mar 2024 19:02:03 +0530 Subject: [PATCH 53/91] api logic updated for feeback --- dev.py | 81 +++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/dev.py b/dev.py index 4e6c777..62338ac 100644 --- a/dev.py +++ b/dev.py @@ -728,7 +728,6 @@ async def store_data(payload: dict, tenant: str = Header(None)): print("l2_matched: ",l2_matched) # Finding the attribute in the global collection - pipeline = [ { "$match": { @@ -754,32 +753,50 @@ async def store_data(payload: dict, tenant: str = Header(None)): global_docs = synonyms_collection.aggregate(pipeline) - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - new_score = synonyms.get(attribute_name, {}).get("score",1) - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, - { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) + if not global_docs: # If no documents found, insert a new document + new_synonym = { + "synonym": attribute_name, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched}": new_synonym + } + }, + upsert=True + ) + print(f"Inserted new synonym: {new_synonym}") - logging.debug(f"Updated score for {attribute_name} to {new_score}") - else: - print("No 'synonyms' found in the document.") + else: + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + print("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): attribute_name = policy1.get("attributeName").lower() @@ -790,7 +807,6 @@ async def store_data(payload: dict, tenant: str = Header(None)): print("l2_matched: ",l2_matched) # Finding the attribute in the global collection - pipeline = [ { "$match": { @@ -817,10 +833,17 @@ async def store_data(payload: dict, tenant: str = Header(None)): global_docs = synonyms_collection.aggregate(pipeline) for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) if synonyms: - # Accessing the score and updating it - new_score = synonyms.get(attribute_name, {}).get("score",1) - 0.2 + + score = global_doc['synonyms']['score'] + + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + # Updating the global collection with the new score synonyms_collection.update_one( { From 9f1dce504f3e8fc3fd3b25d4bd0918dacbbc984a Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 11:24:47 +0530 Subject: [PATCH 54/91] logic updated for user_data extraction from response --- dev.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/dev.py b/dev.py index 62338ac..65c1bb8 100644 --- a/dev.py +++ b/dev.py @@ -124,31 +124,37 @@ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): #-----------------------------extracting the user object from response----------------- def extract_user_data(response): try: - logging.debug(f"extracting the users from the nested json") + logging.debug("Extracting the users from the nested json") user_data_list = [] def is_user_data(obj): - # Check if object contains at least one of the common user data keys - user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} - return any(key in obj for key in user_keys) - - def traverse(obj): + # Convert keys in the object to lowercase for case-insensitive comparison + lower_case_obj_keys = {key.lower() for key in obj.keys()} + # Define user data keys in lowercase + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + # Check if object contains at least one of the common user data keys, ignoring case + return any(key in lower_case_obj_keys for key in user_keys) + + def traverse(obj, original_keys=None): # Recursively traverse the JSON object - nonlocal user_data_list if isinstance(obj, dict): if is_user_data(obj): - user_data_list.append(obj) + # Convert keys in the user data to lowercase or handle as needed + # This step might need adjustments based on how you want to use the extracted data + user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) else: - for value in obj.values(): - traverse(value) + for key, value in obj.items(): + # Maintain original keys for nested dictionaries + traverse(value, {**original_keys, **{key.lower(): key}}) elif isinstance(obj, list): for item in obj: - traverse(item) + traverse(item, original_keys) - traverse(response) + traverse(response, {}) return user_data_list except Exception as e: + logging.error(f"Error extracting user data: {e}") # It's good to log the error for debugging purposes raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #---------------------------extracting keys, datatype, label and jsonpath---------------- @@ -503,7 +509,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #End of changes by Abhishek json_data_ = extract_user_data(json_data) - #print("json_data: ",json_data_) + print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) From 61182aa0afb5d12e2934662483336a54da3b9a8f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 11:26:01 +0530 Subject: [PATCH 55/91] logic updated for user_data extraction from response --- policy_mapping.py | 277 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 211 insertions(+), 66 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 8a69b35..32efec6 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -52,7 +52,6 @@ def stored_response(tenant: str): def stored_policy_mapped(tenant: str): return get_collection(tenant, "amaya_policyMap") - #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): return get_collection(tenant, "amaya_final_policyMap") @@ -125,31 +124,37 @@ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): #-----------------------------extracting the user object from response----------------- def extract_user_data(response): try: - logging.debug(f"extracting the users from the nested json") + logging.debug("Extracting the users from the nested json") user_data_list = [] def is_user_data(obj): - # Check if object contains at least one of the common user data keys - user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} - return any(key in obj for key in user_keys) - - def traverse(obj): + # Convert keys in the object to lowercase for case-insensitive comparison + lower_case_obj_keys = {key.lower() for key in obj.keys()} + # Define user data keys in lowercase + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + # Check if object contains at least one of the common user data keys, ignoring case + return any(key in lower_case_obj_keys for key in user_keys) + + def traverse(obj, original_keys=None): # Recursively traverse the JSON object - nonlocal user_data_list if isinstance(obj, dict): if is_user_data(obj): - user_data_list.append(obj) + # Convert keys in the user data to lowercase or handle as needed + # This step might need adjustments based on how you want to use the extracted data + user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) else: - for value in obj.values(): - traverse(value) + for key, value in obj.items(): + # Maintain original keys for nested dictionaries + traverse(value, {**original_keys, **{key.lower(): key}}) elif isinstance(obj, list): for item in obj: - traverse(item) + traverse(item, original_keys) - traverse(response) + traverse(response, {}) return user_data_list except Exception as e: + logging.error(f"Error extracting user data: {e}") # It's good to log the error for debugging purposes raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #---------------------------extracting keys, datatype, label and jsonpath---------------- @@ -504,7 +509,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #End of changes by Abhishek json_data_ = extract_user_data(json_data) - #print("json_data: ",json_data_) + print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) @@ -657,7 +662,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") #------- Api for body populating---------- -@app.post("/generativeaisrvc/map_fields_to_policy/") +@app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): try: body = payload.get("body") @@ -692,69 +697,209 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- -# @app.post("/generativeaisrvc/store_data") -# async def store_data(payload: dict, tenant: str = Header(None)): -# try: -# request_Id = payload.get("request_id") -# policymap_colection = stored_admin_policymap(tenant) -# policymap_colection.insert_one(payload) - -# logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") +@app.post("/generativeaisrvc/store_data") +async def store_data(payload: dict, tenant: str = Header(None)): + print("tenant: ", tenant) + try: + request_Id = payload.get("request_id") + policymap_collection = stored_admin_policymap(tenant) + policymap_collection.insert_one(payload) -# # query AI suggestion collection -# subset_collection = stored_policy_mapped(tenant) -# doc1 = subset_collection.find_one(request_Id) + logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") -# # query admin collection -# doc2 = policymap_colection.find_one(request_Id) + # query AI suggestion collection + subset_collection = stored_policy_mapped(tenant) + doc1 = subset_collection.find_one({"request_id":request_Id}) -# #query global collection -# synonyms_collection = get_master_collection("amayaSynonymsMaster") + # query admin collection + doc2 = policymap_collection.find_one({"request_id":request_Id}) -# if doc1 and doc2: -# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): -# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #query global collection + synonyms_collection = get_master_collection("amayaSynonymsMaster") -# #add logic if synonyms not present add new synonyms to respective key -# # Fetch and update the global collection document -# l2_matched = policy1.get("l2_matched") -# global_doc = synonyms_collection.find_one({"_id": l2_matched}) -# if global_doc: -# new_score = global_doc.get("score", 1) - 0.2 -# synonyms_collection.update_one({"_id": l2_matched}, {"$set": {"score": new_score}}) -# print(f"Updated score for {l2_matched} to {new_score}") -# else: -# print("Documents with the given request_id not found in one or both collections.") + if doc1 and doc2: + # print("doc1: ",doc1) + # print("doc2: ",doc2) + for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): + # print("policy1: ",policy1) + # print("policy2: ",policy2) + + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #Fetching attributeName from doc1 + attribute_name = policy1.get("attributeName").lower() + print("attribute_name: ",attribute_name) + + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + print("l2_matched: ",l2_matched) + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + global_docs = synonyms_collection.aggregate(pipeline) + if not global_docs: # If no documents found, insert a new document + new_synonym = { + "synonym": attribute_name, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched}": new_synonym + } + }, + upsert=True + ) + print(f"Inserted new synonym: {new_synonym}") -# if doc1 and doc2: -# for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): -# if policy1.get("matching_condition") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): -# # Fetching attributeName from doc1 -# attribute_name = doc1.get("attributeName") - -# # Fetching l2_matched from doc1 -# l2_matched = policy1.get("l2_matched") + else: + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + print("No 'synonyms' found in the document.") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + attribute_name = policy1.get("attributeName").lower() + print("attribute_name: ",attribute_name) -# # Finding the attribute in the global collection -# global_doc = synonyms_collection.find_one({"synonyms.{}".format(attribute_name): {"$exists": True}}) + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + print("l2_matched: ",l2_matched) -# if global_doc: -# new_score = global_doc.get("synonyms", {}).get(attribute_name, {}).get("score", 1) - 0.2 - -# # Updating the global collection with the new score -# synonyms_collection.update_one({"_id": global_doc["_id"], "synonyms.{}".format(attribute_name): {"$exists": True}}, -# {"$set": {"synonyms.{}.score".format(attribute_name): new_score}}) - -# print(f"Updated score for {attribute_name} to {new_score}") - - -# #compare fields and make calculation to update the in global collection -# return {"message": "Data saved successfully"} + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + + synonyms = global_doc.get("synonyms", {}) + if synonyms: + + score = global_doc['synonyms']['score'] + + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + print("No 'synonyms' found in the document.") + + else: + print("failed") + + #compare fields and make calculation to update the in global collection + return {"message": "Data saved successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +#------------ API for mapping roles------------- +# @app.post("/generativeaisrvc/map_fields_to_roles/") +# async def map_fields_to_roles(payload: Dict[str, Any]): +# try: +# body = payload.get("body") +# policy_mapping = payload.get("policyMapping") + +# if not body: +# raise HTTPException(status_code=400, detail="body empty") +# elif not policy_mapping: +# raise HTTPException(status_code=400, detail="policy_mapping empty") + +# except HTTPException: +# raise # except Exception as e: -# raise HTTPException(status_code=500, detail=str(e)) +# return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From a24830b99f6546b1e390bf7229ed7e49021c7ac0 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 12:48:49 +0530 Subject: [PATCH 56/91] removed extra spaces from datatype for login which populated the datatype in policy mapped --- dev.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev.py b/dev.py index 65c1bb8..1d8af01 100644 --- a/dev.py +++ b/dev.py @@ -544,7 +544,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): 'lastName': 'STRING', 'userType': 'STRING', 'end_date': 'DATE', - 'login': 'STRING ', + 'login': 'STRING', 'userType': 'STRING', 'dateOfBirth': 'DATE', 'endDate': 'DATE', @@ -570,9 +570,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): threshold = 60 appId = data.get("appId") - #--- Inserting the synonyms_dict for appId-------- - #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) #print("result: ", result) @@ -662,7 +659,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") #------- Api for body populating---------- -@app.post("/generativeaisrvc/map_fields_to_policy/") +@app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): try: body = payload.get("body") @@ -882,7 +879,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): #------------ API for mapping roles------------- -# @app.post("/generativeaisrvc/map_fields_to_roles/") +# @app.post("/generativeaisrvc/map_fields_to_roles") # async def map_fields_to_roles(payload: Dict[str, Any]): # try: # body = payload.get("body") From 0bedcc595b2fb3b4fc7171900045e5dd49be0fd7 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 12:49:01 +0530 Subject: [PATCH 57/91] removed extra spaces from datatype for login which populated the datatype in policy mapped --- policy_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policy_mapping.py b/policy_mapping.py index 32efec6..8500924 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -544,7 +544,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): 'lastName': 'STRING', 'userType': 'STRING', 'end_date': 'DATE', - 'login': 'STRING ', + 'login': 'STRING', 'userType': 'STRING', 'dateOfBirth': 'DATE', 'endDate': 'DATE', From e5699c7508e426818b83aea33c2b7e3b56dbf2fb Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 16:41:49 +0530 Subject: [PATCH 58/91] logic added for accepting payload as list and dict both --- dev.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/dev.py b/dev.py index 1d8af01..f1d0c1f 100644 --- a/dev.py +++ b/dev.py @@ -90,6 +90,11 @@ def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] +#--------------for preprocess purpose +def remove_underscores_from_set(input_set): + return set(element.replace('_', '') for element in input_set) + + #--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() @@ -140,7 +145,6 @@ def traverse(obj, original_keys=None): if isinstance(obj, dict): if is_user_data(obj): # Convert keys in the user data to lowercase or handle as needed - # This step might need adjustments based on how you want to use the extracted data user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) else: for key, value in obj.items(): @@ -154,7 +158,7 @@ def traverse(obj, original_keys=None): return user_data_list except Exception as e: - logging.error(f"Error extracting user data: {e}") # It's good to log the error for debugging purposes + logging.error(f"Error extracting user data: {e}") raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #---------------------------extracting keys, datatype, label and jsonpath---------------- @@ -453,7 +457,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str if not matched: print(f"No match found for '{field}'") - return field, None # Return original field if no match is found + return field, None #------- works on nested conditions also @@ -500,16 +504,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Validate the format of 'payload' if not isinstance(data['payload'], dict): - raise HTTPException(status_code=400, detail="'payload' must be a dictionary") - + if isinstance(data['payload'], list): + # Convert list of dictionaries to a single dictionary + converted_payload = {} + for item in data['payload']: + for key, value in item.items(): + converted_payload[key] = value + data['payload'] = converted_payload + else: + raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") json_data = data.get('payload') #print("json data is {}", json_data) - #End of changes by Abhishek json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) + #print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) @@ -520,10 +530,12 @@ async def get_mapped(data: dict, tenant: str = Header(...)): if isinstance(l1, str): l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) + print("list1 as str: ",l1_list) else: - l1_list = set(l1) - print("list1: ",l1_list) + l1_list = remove_underscores_from_set(l1) + l1_list = set(l1_list) + print("list1 as set: ",l1_list) + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] From d03d304b564e147a2c16759ac9afc6faca962e32 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 16:42:01 +0530 Subject: [PATCH 59/91] logic added for accepting payload as list and dict both --- policy_mapping.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 8500924..f1d0c1f 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -90,6 +90,11 @@ def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] +#--------------for preprocess purpose +def remove_underscores_from_set(input_set): + return set(element.replace('_', '') for element in input_set) + + #--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() @@ -140,7 +145,6 @@ def traverse(obj, original_keys=None): if isinstance(obj, dict): if is_user_data(obj): # Convert keys in the user data to lowercase or handle as needed - # This step might need adjustments based on how you want to use the extracted data user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) else: for key, value in obj.items(): @@ -154,7 +158,7 @@ def traverse(obj, original_keys=None): return user_data_list except Exception as e: - logging.error(f"Error extracting user data: {e}") # It's good to log the error for debugging purposes + logging.error(f"Error extracting user data: {e}") raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #---------------------------extracting keys, datatype, label and jsonpath---------------- @@ -453,7 +457,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str if not matched: print(f"No match found for '{field}'") - return field, None # Return original field if no match is found + return field, None #------- works on nested conditions also @@ -500,16 +504,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Validate the format of 'payload' if not isinstance(data['payload'], dict): - raise HTTPException(status_code=400, detail="'payload' must be a dictionary") - + if isinstance(data['payload'], list): + # Convert list of dictionaries to a single dictionary + converted_payload = {} + for item in data['payload']: + for key, value in item.items(): + converted_payload[key] = value + data['payload'] = converted_payload + else: + raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") json_data = data.get('payload') #print("json data is {}", json_data) - #End of changes by Abhishek json_data_ = extract_user_data(json_data) - print("json_data: ",json_data_) + #print("json_data: ",json_data_) response_data = get_distinct_keys_and_datatypes(json_data_) #response_data=list(response_data.values()) @@ -520,10 +530,12 @@ async def get_mapped(data: dict, tenant: str = Header(...)): if isinstance(l1, str): l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) + print("list1 as str: ",l1_list) else: - l1_list = set(l1) - print("list1: ",l1_list) + l1_list = remove_underscores_from_set(l1) + l1_list = set(l1_list) + print("list1 as set: ",l1_list) + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] @@ -570,9 +582,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): threshold = 60 appId = data.get("appId") - #--- Inserting the synonyms_dict for appId-------- - #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) #print("result: ", result) @@ -882,7 +891,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): #------------ API for mapping roles------------- -# @app.post("/generativeaisrvc/map_fields_to_roles/") +# @app.post("/generativeaisrvc/map_fields_to_roles") # async def map_fields_to_roles(payload: Dict[str, Any]): # try: # body = payload.get("body") From ec0a401593fd9d8b0546722035a82322962a236d Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 18:15:00 +0530 Subject: [PATCH 60/91] logic updated for matching fields for felicity app --- dev.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev.py b/dev.py index f1d0c1f..e91e07d 100644 --- a/dev.py +++ b/dev.py @@ -94,7 +94,6 @@ def convert_string_to_list(input_str: str) -> List[str]: def remove_underscores_from_set(input_set): return set(element.replace('_', '') for element in input_set) - #--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() @@ -532,8 +531,9 @@ async def get_mapped(data: dict, tenant: str = Header(...)): l1_list = set(convert_string_to_list(l1)) print("list1 as str: ",l1_list) else: - l1_list = remove_underscores_from_set(l1) - l1_list = set(l1_list) + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) print("list1 as set: ",l1_list) From 93dcd30b11222ffe5f5bc32ba8b62888284b945f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 28 Mar 2024 18:15:07 +0530 Subject: [PATCH 61/91] logic updated for matching fields for felicity app --- policy_mapping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index f1d0c1f..e91e07d 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -94,7 +94,6 @@ def convert_string_to_list(input_str: str) -> List[str]: def remove_underscores_from_set(input_set): return set(element.replace('_', '') for element in input_set) - #--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() @@ -532,8 +531,9 @@ async def get_mapped(data: dict, tenant: str = Header(...)): l1_list = set(convert_string_to_list(l1)) print("list1 as str: ",l1_list) else: - l1_list = remove_underscores_from_set(l1) - l1_list = set(l1_list) + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) print("list1 as set: ",l1_list) From f69b8e125f70a86792d59176d59feca94713baf8 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 1 Apr 2024 12:56:51 +0530 Subject: [PATCH 62/91] added logging statements for debugging --- dev.py | 294 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 148 insertions(+), 146 deletions(-) diff --git a/dev.py b/dev.py index e91e07d..4d7d3c3 100644 --- a/dev.py +++ b/dev.py @@ -411,7 +411,7 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo }) processed_labels.add(element['element_name_l1']) else: - print(f"No matched data found for {element['element_name_l1']}") + logging.debug(f"No matched data found for {element['element_name_l1']}") for data in response_data: @@ -442,7 +442,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str internal_field = map_entry["internal"] if external_field.lower() == field.lower(): matched = True - print(f"Exact match found: '{field}' -> '{external_field}'") + logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found @@ -451,11 +451,11 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str for map_entry in policy_mapping: if map_entry["internal"].lower() == best_match: matched = True - print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: - print(f"No match found for '{field}'") + logging.debug(f"No match found for '{field}'") return field, None @@ -479,7 +479,6 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): - print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: @@ -518,151 +517,156 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #print("json data is {}", json_data) json_data_ = extract_user_data(json_data) - #print("json_data: ",json_data_) - response_data = get_distinct_keys_and_datatypes(json_data_) - #response_data=list(response_data.values()) - #print("response_data:", response_data) + if json_data_: + logging.info(f" Successfully extract the json data from response") + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + #print("response_data:", response_data) - l1 = [item['label'] for item in response_data] - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1 as str: ",l1_list) - else: - #l1_list = remove_underscores_from_set(l1) - l1_list = set(l1) - #l1_list = set(l1_list) - print("list1 as set: ",l1_list) - - - - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] - - l2_datatypes = { - 'department': 'STRING', - 'employeeId': 'STRING', - 'designation': 'STRING', - 'appUpdatedDate': 'DATETIME', - 'displayName': 'STRING', - 'mobile': 'STRING', - 'country': 'STRING', - 'city': 'STRING', - 'email': 'STRING', - 'end_date': 'DATE', - 'firstName': 'STRING', - 'login': 'STRING', - 'lastName': 'STRING', - 'userType': 'STRING', - 'end_date': 'DATE', - 'login': 'STRING', - 'userType': 'STRING', - 'dateOfBirth': 'DATE', - 'endDate': 'DATE', - 'startDate': 'DATE', - 'password': 'password', - 'status': 'STRING', - 'profilePicture': 'profilePicture', - 'appUserId': 'STRING', - 'landline': 'STRING' - } - - l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + l1 = [item['label'] for item in response_data] - print("list2: ",l2) + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + logging.info(f"list1: {l1_list}") + else: + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) + logging.info(f"list1: {l1_list}") + + + + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + + l2_datatypes = { + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'login': 'STRING', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'STRING', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' + } + + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) - #print("custom_attributes: ", custom_attributes) - - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 + logging.info(f"list 2: {l2}") + + #print("custom_attributes: ", custom_attributes) + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 - threshold = 60 - appId = data.get("appId") + threshold = 60 + appId = data.get("appId") - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) - #print("result: ", result) - - request_id = generate_request_id() - - score_collection = stored_score(tenant, appId) - - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) - #print("final response: ",final_response) - final_response_dict = {"final_response": final_response} - - # Assuming 'appId' is present in the received response - final_response_dict['appId'] = appId - - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) - - logging.debug("Final response saved successfully") - - subset_response = output_collection.aggregate([ - {"$unwind": "$final_response"}, - {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, - {"$group": { - "_id": "$final_response.attributeName", - "data": {"$first": "$final_response"} - }}, - {"$project": { - "_id": 0, - "jsonPath": "$data.jsonPath", - "attributeName": "$data.attributeName", - "l1_datatype": "$data.l1_datatype", - "l2_matched": "$data.l2_matched", - "l2_datatype": "$data.l2_datatype", - "value": "$data.value", - "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence", - "matching_decision": "$data.matching_decision", - "isCustom": "$data.isCustom" - }} - ]) - - - subset_response_data = list(subset_response) - # Serialize each document into a JSON serializable format - json_serializable_response = [] - for doc in subset_response_data: - json_serializable_doc = { - "jsonPath": doc["jsonPath"], - "attributeName": doc["attributeName"], - "l1_datatype": doc["l1_datatype"], - "l2_matched": doc["l2_matched"], - "l2_datatype": doc["l2_datatype"], - "value": doc["value"], - "similarity_percentage": doc["similarity_percentage"], - "confidence": doc["confidence"], - "matching_decision": doc["matching_decision"], - "isCustom": doc["isCustom"] + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + #print("result: ", result) + + request_id = generate_request_id() + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) + #print("final response: ",final_response) + final_response_dict = {"final_response": final_response} + + # Assuming 'appId' is present in the received response + final_response_dict['appId'] = appId + + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { + "_id": "$final_response.attributeName", + "data": {"$first": "$final_response"} + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence", + "matching_decision": "$data.matching_decision", + "isCustom": "$data.isCustom" + }} + ]) + + + subset_response_data = list(subset_response) + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"], + "matching_decision": doc["matching_decision"], + "isCustom": doc["isCustom"] + } + json_serializable_response.append(json_serializable_doc) + + + aggregated_data = { + "appId": appId, + "request_id": request_id, + "policyMapList": subset_response_data } - json_serializable_response.append(json_serializable_doc) + subset_collection.insert_one(aggregated_data) + + logging.debug("subset response saved successfully") - aggregated_data = { - "appId": appId, + data_response = { "request_id": request_id, - "policyMapList": subset_response_data + "content": json_serializable_response } - subset_collection.insert_one(aggregated_data) - - logging.debug("subset response saved successfully") - - data_response = { - "request_id": request_id, - "content": json_serializable_response - } - - #return JSONResponse(content=json_serializable_response) - return ResponseModel(data=data_response, message="Policy mapping generated successfully") + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + + else: + logging.info(f" Failed to extract the data from the response") except HTTPException: raise @@ -708,7 +712,6 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/store_data") async def store_data(payload: dict, tenant: str = Header(None)): - print("tenant: ", tenant) try: request_Id = payload.get("request_id") policymap_collection = stored_admin_policymap(tenant) @@ -736,11 +739,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): #Fetching attributeName from doc1 attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) + logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ",l2_matched) + logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -782,7 +785,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - print(f"Inserted new synonym: {new_synonym}") + logging.debug(f"Inserted new synonym: {new_synonym}") else: for global_doc in global_docs: @@ -811,15 +814,15 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"Updated score for {attribute_name} to {new_score}") else: - print("No 'synonyms' found in the document.") + logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) + logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ",l2_matched) + logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -879,10 +882,10 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"Updated score for {attribute_name} to {new_score}") else: - print("No 'synonyms' found in the document.") + logging.debug("No 'synonyms' found in the document.") else: - print("failed") + logging.debug("failed") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} @@ -910,5 +913,4 @@ async def store_data(payload: dict, tenant: str = Header(None)): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From cd9bb0d0b4176d77cfdd073b14e72a107c198505 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 1 Apr 2024 15:14:46 +0530 Subject: [PATCH 63/91] added additional logging statement for policy_mapping.py file --- policy_mapping.py | 294 +++++++++++++++++++++++----------------------- 1 file changed, 148 insertions(+), 146 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index e91e07d..4d7d3c3 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -411,7 +411,7 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo }) processed_labels.add(element['element_name_l1']) else: - print(f"No matched data found for {element['element_name_l1']}") + logging.debug(f"No matched data found for {element['element_name_l1']}") for data in response_data: @@ -442,7 +442,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str internal_field = map_entry["internal"] if external_field.lower() == field.lower(): matched = True - print(f"Exact match found: '{field}' -> '{external_field}'") + logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found @@ -451,11 +451,11 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str for map_entry in policy_mapping: if map_entry["internal"].lower() == best_match: matched = True - print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: - print(f"No match found for '{field}'") + logging.debug(f"No match found for '{field}'") return field, None @@ -479,7 +479,6 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): - print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: @@ -518,151 +517,156 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #print("json data is {}", json_data) json_data_ = extract_user_data(json_data) - #print("json_data: ",json_data_) - response_data = get_distinct_keys_and_datatypes(json_data_) - #response_data=list(response_data.values()) - #print("response_data:", response_data) + if json_data_: + logging.info(f" Successfully extract the json data from response") + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + #print("response_data:", response_data) - l1 = [item['label'] for item in response_data] - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1 as str: ",l1_list) - else: - #l1_list = remove_underscores_from_set(l1) - l1_list = set(l1) - #l1_list = set(l1_list) - print("list1 as set: ",l1_list) - - - - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] - - l2_datatypes = { - 'department': 'STRING', - 'employeeId': 'STRING', - 'designation': 'STRING', - 'appUpdatedDate': 'DATETIME', - 'displayName': 'STRING', - 'mobile': 'STRING', - 'country': 'STRING', - 'city': 'STRING', - 'email': 'STRING', - 'end_date': 'DATE', - 'firstName': 'STRING', - 'login': 'STRING', - 'lastName': 'STRING', - 'userType': 'STRING', - 'end_date': 'DATE', - 'login': 'STRING', - 'userType': 'STRING', - 'dateOfBirth': 'DATE', - 'endDate': 'DATE', - 'startDate': 'DATE', - 'password': 'password', - 'status': 'STRING', - 'profilePicture': 'profilePicture', - 'appUserId': 'STRING', - 'landline': 'STRING' - } - - l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + l1 = [item['label'] for item in response_data] - print("list2: ",l2) + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + logging.info(f"list1: {l1_list}") + else: + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) + logging.info(f"list1: {l1_list}") + + + + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + + l2_datatypes = { + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'login': 'STRING', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'STRING', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' + } + + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) - #print("custom_attributes: ", custom_attributes) - - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 + logging.info(f"list 2: {l2}") + + #print("custom_attributes: ", custom_attributes) + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 - threshold = 60 - appId = data.get("appId") + threshold = 60 + appId = data.get("appId") - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) - #print("result: ", result) - - request_id = generate_request_id() - - score_collection = stored_score(tenant, appId) - - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) - #print("final response: ",final_response) - final_response_dict = {"final_response": final_response} - - # Assuming 'appId' is present in the received response - final_response_dict['appId'] = appId - - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) - - logging.debug("Final response saved successfully") - - subset_response = output_collection.aggregate([ - {"$unwind": "$final_response"}, - {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, - {"$group": { - "_id": "$final_response.attributeName", - "data": {"$first": "$final_response"} - }}, - {"$project": { - "_id": 0, - "jsonPath": "$data.jsonPath", - "attributeName": "$data.attributeName", - "l1_datatype": "$data.l1_datatype", - "l2_matched": "$data.l2_matched", - "l2_datatype": "$data.l2_datatype", - "value": "$data.value", - "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence", - "matching_decision": "$data.matching_decision", - "isCustom": "$data.isCustom" - }} - ]) - - - subset_response_data = list(subset_response) - # Serialize each document into a JSON serializable format - json_serializable_response = [] - for doc in subset_response_data: - json_serializable_doc = { - "jsonPath": doc["jsonPath"], - "attributeName": doc["attributeName"], - "l1_datatype": doc["l1_datatype"], - "l2_matched": doc["l2_matched"], - "l2_datatype": doc["l2_datatype"], - "value": doc["value"], - "similarity_percentage": doc["similarity_percentage"], - "confidence": doc["confidence"], - "matching_decision": doc["matching_decision"], - "isCustom": doc["isCustom"] + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + #print("result: ", result) + + request_id = generate_request_id() + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) + #print("final response: ",final_response) + final_response_dict = {"final_response": final_response} + + # Assuming 'appId' is present in the received response + final_response_dict['appId'] = appId + + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { + "_id": "$final_response.attributeName", + "data": {"$first": "$final_response"} + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence", + "matching_decision": "$data.matching_decision", + "isCustom": "$data.isCustom" + }} + ]) + + + subset_response_data = list(subset_response) + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"], + "matching_decision": doc["matching_decision"], + "isCustom": doc["isCustom"] + } + json_serializable_response.append(json_serializable_doc) + + + aggregated_data = { + "appId": appId, + "request_id": request_id, + "policyMapList": subset_response_data } - json_serializable_response.append(json_serializable_doc) + subset_collection.insert_one(aggregated_data) + + logging.debug("subset response saved successfully") - aggregated_data = { - "appId": appId, + data_response = { "request_id": request_id, - "policyMapList": subset_response_data + "content": json_serializable_response } - subset_collection.insert_one(aggregated_data) - - logging.debug("subset response saved successfully") - - data_response = { - "request_id": request_id, - "content": json_serializable_response - } - - #return JSONResponse(content=json_serializable_response) - return ResponseModel(data=data_response, message="Policy mapping generated successfully") + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + + else: + logging.info(f" Failed to extract the data from the response") except HTTPException: raise @@ -708,7 +712,6 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/store_data") async def store_data(payload: dict, tenant: str = Header(None)): - print("tenant: ", tenant) try: request_Id = payload.get("request_id") policymap_collection = stored_admin_policymap(tenant) @@ -736,11 +739,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): #Fetching attributeName from doc1 attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) + logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ",l2_matched) + logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -782,7 +785,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - print(f"Inserted new synonym: {new_synonym}") + logging.debug(f"Inserted new synonym: {new_synonym}") else: for global_doc in global_docs: @@ -811,15 +814,15 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"Updated score for {attribute_name} to {new_score}") else: - print("No 'synonyms' found in the document.") + logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) + logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ",l2_matched) + logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -879,10 +882,10 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"Updated score for {attribute_name} to {new_score}") else: - print("No 'synonyms' found in the document.") + logging.debug("No 'synonyms' found in the document.") else: - print("failed") + logging.debug("failed") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} @@ -910,5 +913,4 @@ async def store_data(payload: dict, tenant: str = Header(None)): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From f2725e712ff036403faf85a21b59d5b2cba68ac3 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 1 Apr 2024 19:25:48 +0530 Subject: [PATCH 64/91] files updated with feedback logic --- dev.py | 141 +++++++++++++++++++++++++++++----------------- policy_mapping.py | 18 ++++-- 2 files changed, 102 insertions(+), 57 deletions(-) diff --git a/dev.py b/dev.py index 4d7d3c3..94b7694 100644 --- a/dev.py +++ b/dev.py @@ -737,19 +737,20 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating where policy1 and policy2 are not equal") #Fetching attributeName from doc1 - attribute_name = policy1.get("attributeName").lower() - logging.debug("attribute_name: ",attribute_name) + attribute_name1 = policy1.get("attributeName").lower() + print("attribute_name1: ",attribute_name1) # Fetching l2_matched from doc1 - l2_matched = policy1.get("l2_matched") - logging.debug("l2_matched: ",l2_matched) + l2_matched1 = policy1.get("l2_matched") + print("l2_matched1: ",l2_matched1) # Finding the attribute in the global collection pipeline = [ { "$match": { - f"synonyms.{l2_matched}.synonym": attribute_name + f"synonyms.{l2_matched1}.synonym": attribute_name } }, { @@ -757,7 +758,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): "_id": 0, "synonyms": { "$filter": { - "input": f"$synonyms.{l2_matched}", + "input": f"$synonyms.{l2_matched1}", "as": "item", "cond": { "$eq": ["$$item.synonym", attribute_name] } } @@ -771,58 +772,68 @@ async def store_data(payload: dict, tenant: str = Header(None)): global_docs = synonyms_collection.aggregate(pipeline) - if not global_docs: # If no documents found, insert a new document - new_synonym = { - "synonym": attribute_name, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched}": new_synonym - } - }, - upsert=True - ) - logging.debug(f"Inserted new synonym: {new_synonym}") - - else: - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - score = global_doc['synonyms']['score'] - new_score = score - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) - - logging.debug(f"Updated score for {attribute_name} to {new_score}") - else: - logging.debug("No 'synonyms' found in the document.") + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score}") + else: + logging.debug("No 'synonyms' found in the document.") + + + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name2: ",attribute_name2) + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched2: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating where policy1 and policy2 are equal") attribute_name = policy1.get("attributeName").lower() - logging.debug("attribute_name: ",attribute_name) + print("attribute_name: ",attribute_name) + #logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - logging.debug("l2_matched: ",l2_matched) + print("l2_matched: ", l2_matched) + #logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -884,8 +895,34 @@ async def store_data(payload: dict, tenant: str = Header(None)): else: logging.debug("No 'synonyms' found in the document.") + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty") + + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name2: ",attribute_name2) + + # Fetching l2_matched from doc1 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched2: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}") + + else: - logging.debug("failed") + logging.debug("no need to analyze and changed") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} diff --git a/policy_mapping.py b/policy_mapping.py index 4d7d3c3..47d885b 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -737,13 +737,18 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating where policy1 and policy2 are not equal") #Fetching attributeName from doc1 attribute_name = policy1.get("attributeName").lower() - logging.debug("attribute_name: ",attribute_name) + print("attribute_name: ",attribute_name) + #logging.debug("attribute_name: ",attribute_name) + #logging.debug("attribute_name: %s", attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - logging.debug("l2_matched: ",l2_matched) + print("l2_matched: ",l2_matched) + #logging.debug("l2_matched: ",l2_matched) + #logging.debug("l2_matched: %s", l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -817,12 +822,15 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating where policy1 and policy2 are equal") attribute_name = policy1.get("attributeName").lower() - logging.debug("attribute_name: ",attribute_name) + print("attribute_name: ",attribute_name) + #logging.debug("attribute_name: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - logging.debug("l2_matched: ",l2_matched) + print("l2_matched: ", l2_matched) + #logging.debug("l2_matched: ",l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -885,7 +893,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug("No 'synonyms' found in the document.") else: - logging.debug("failed") + logging.debug("no need to analyze and changed") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} From e195b3f064b0bb99ebcdbd62e611b527c4fcde58 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 2 Apr 2024 12:25:38 +0530 Subject: [PATCH 65/91] updated logics for feedback api --- dev.py | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/dev.py b/dev.py index 94b7694..15320de 100644 --- a/dev.py +++ b/dev.py @@ -713,6 +713,12 @@ async def map_fields_to_policy(payload: Dict[str, Any]): @app.post("/generativeaisrvc/store_data") async def store_data(payload: dict, tenant: str = Header(None)): try: + # Check if 'request_id' and 'payload' are present in the request + if 'request_Id' not in payload: + raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") + elif 'payload' not in payload: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + request_Id = payload.get("request_id") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) @@ -737,20 +743,20 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating where policy1 and policy2 are not equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() - print("attribute_name1: ",attribute_name1) + print("attribute_name of the application: ",attribute_name1) # Fetching l2_matched from doc1 l2_matched1 = policy1.get("l2_matched") - print("l2_matched1: ",l2_matched1) + print("l2_matched suggested by AI: ",l2_matched1) # Finding the attribute in the global collection pipeline = [ { "$match": { - f"synonyms.{l2_matched1}.synonym": attribute_name + f"synonyms.{l2_matched1}.synonym": attribute_name1 } }, { @@ -760,7 +766,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): "$filter": { "input": f"$synonyms.{l2_matched1}", "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name] } + "cond": { "$eq": ["$$item.synonym", attribute_name1] } } } } @@ -781,7 +787,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Updating the global collection with the new score synonyms_collection.update_one( { - f"synonyms.{l2_matched1}.synonym": str(attribute_name) + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) }, { "$set": { @@ -790,27 +796,26 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, array_filters=[ { - "elem.synonym": str(attribute_name) + "elem.synonym": str(attribute_name1) } ], upsert= True ) - logging.debug(f"Updated score for {attribute_name} to {new_score}") + logging.debug(f"Updated score for {attribute_name1} to {new_score} since the suggestion given was wrong by AI") else: logging.debug("No 'synonyms' found in the document.") - #----------------------for storing new synonyms against the admin l2matched--------------------- attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name2: ",attribute_name2) + print("attribute_name of the application: ",attribute_name2) # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched2: ",l2_matched2) + print("l2_matched by admin: ",l2_matched2) new_synonym = { - "synonym": attribute_name, + "synonym": attribute_name2, "score": 1 } synonyms_collection.update_one( @@ -822,18 +827,17 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - logging.debug(f"Inserted new synonym: {new_synonym}") + + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating where policy1 and policy2 are equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal") attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) - #logging.debug("attribute_name: ",attribute_name) + print("attribute_name of the application: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ", l2_matched) - #logging.debug("l2_matched: ",l2_matched) + print("l2_matched suggested by AI: ", l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -891,19 +895,19 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name} to {new_score}") + logging.debug(f"Updated score for {attribute_name} to {new_score} since the suggestion given was right by AI") else: logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty") + logging.debug(f" checking and updating where matching decision is empty string") attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name2: ",attribute_name2) + print("attribute_name of the application: ",attribute_name2) - # Fetching l2_matched from doc1 + # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched2: ",l2_matched2) + print("l2_matched by admin: ",l2_matched2) new_synonym = { "synonym": attribute_name2, From 792bdc1a66cf87d28439dff0452616eec57f7363 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 2 Apr 2024 12:25:45 +0530 Subject: [PATCH 66/91] updated logics for feedback api --- policy_mapping.py | 179 +++++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 83 deletions(-) diff --git a/policy_mapping.py b/policy_mapping.py index 47d885b..929f8d9 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -710,9 +710,15 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- -@app.post("/generativeaisrvc/store_data") +@app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): try: + # Check if 'request_id' and 'payload' are present in the request + if 'request_Id' not in payload: + raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") + elif 'payload' not in payload: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + request_Id = payload.get("request_id") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) @@ -737,24 +743,20 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating where policy1 and policy2 are not equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal") #Fetching attributeName from doc1 - attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) - #logging.debug("attribute_name: ",attribute_name) - #logging.debug("attribute_name: %s", attribute_name) + attribute_name1 = policy1.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name1) # Fetching l2_matched from doc1 - l2_matched = policy1.get("l2_matched") - print("l2_matched: ",l2_matched) - #logging.debug("l2_matched: ",l2_matched) - #logging.debug("l2_matched: %s", l2_matched) + l2_matched1 = policy1.get("l2_matched") + print("l2_matched suggested by AI: ",l2_matched1) # Finding the attribute in the global collection pipeline = [ { "$match": { - f"synonyms.{l2_matched}.synonym": attribute_name + f"synonyms.{l2_matched1}.synonym": attribute_name1 } }, { @@ -762,9 +764,9 @@ async def store_data(payload: dict, tenant: str = Header(None)): "_id": 0, "synonyms": { "$filter": { - "input": f"$synonyms.{l2_matched}", + "input": f"$synonyms.{l2_matched1}", "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name] } + "cond": { "$eq": ["$$item.synonym", attribute_name1] } } } } @@ -776,61 +778,66 @@ async def store_data(payload: dict, tenant: str = Header(None)): global_docs = synonyms_collection.aggregate(pipeline) - if not global_docs: # If no documents found, insert a new document - new_synonym = { - "synonym": attribute_name, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched}": new_synonym - } - }, - upsert=True - ) - logging.debug(f"Inserted new synonym: {new_synonym}") - - else: - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - score = global_doc['synonyms']['score'] - new_score = score - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) - - logging.debug(f"Updated score for {attribute_name} to {new_score}") - else: - logging.debug("No 'synonyms' found in the document.") + "elem.synonym": str(attribute_name1) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name1} to {new_score} since the suggestion given was wrong by AI") + else: + logging.debug("No 'synonyms' found in the document.") + + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name2) + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched by admin: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating where policy1 and policy2 are equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal") attribute_name = policy1.get("attributeName").lower() - print("attribute_name: ",attribute_name) - #logging.debug("attribute_name: ",attribute_name) + print("attribute_name of the application: ",attribute_name) # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched: ", l2_matched) - #logging.debug("l2_matched: ",l2_matched) + print("l2_matched suggested by AI: ", l2_matched) # Finding the attribute in the global collection pipeline = [ @@ -888,37 +895,43 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name} to {new_score}") + logging.debug(f"Updated score for {attribute_name} to {new_score} since the suggestion given was right by AI") else: logging.debug("No 'synonyms' found in the document.") + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty string") + + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name2) + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched by admin: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}") + + else: logging.debug("no need to analyze and changed") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -#------------ API for mapping roles------------- -# @app.post("/generativeaisrvc/map_fields_to_roles") -# async def map_fields_to_roles(payload: Dict[str, Any]): -# try: -# body = payload.get("body") -# policy_mapping = payload.get("policyMapping") - -# if not body: -# raise HTTPException(status_code=400, detail="body empty") -# elif not policy_mapping: -# raise HTTPException(status_code=400, detail="policy_mapping empty") - -# except HTTPException: -# raise -# except Exception as e: -# return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") - - + raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From f4277c69cd56c88ee20ee4201465ac409ae8146c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 2 Apr 2024 12:26:37 +0530 Subject: [PATCH 67/91] endpoint name changed from store_data to feedback --- routes.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes.yaml b/routes.yaml index 5d667f4..401180f 100644 --- a/routes.yaml +++ b/routes.yaml @@ -140,9 +140,9 @@ spec: methods: - '*' port: 0 - rewrite: /generativeaisrvc/store_data + rewrite: /generativeaisrvc/feedback type: prefix - url: /generativeaisrvc/store_data + url: /generativeaisrvc/feedback targets: - host: generativeaiservice.unotech.svc.cluster.local port: 8080 From b5fbfd4f3474e397bc46b351c09d7d5e01954f3e Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 3 Apr 2024 12:21:08 +0530 Subject: [PATCH 68/91] preprocess the synonyms before inserting into collection --- dev.py | 53 ++++++++++++++++++++++++----------------------- policy_mapping.py | 29 ++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/dev.py b/dev.py index 15320de..5da9274 100644 --- a/dev.py +++ b/dev.py @@ -20,6 +20,7 @@ import uvicorn import uuid from typing import Set +import re app = FastAPI() @@ -252,11 +253,12 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): # Check similarity with original list (l2) for element_l2 in l2: el1 = str(element_l1).lower() + el1 = re.sub(r'[^a-zA-Z0-9]', '', el1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio(el1, el2) - if similarity > max_similarity and similarity >= threshold: - max_similarity = similarity - matching_element_l2 = element_l2 + # similarity = fuzz.ratio(el1, el2) + # if similarity > max_similarity and similarity >= threshold: + # max_similarity = similarity + # matching_element_l2 = element_l2 if not matching_element_l2: synonyms_doc = synonyms_collection.find_one() @@ -571,6 +573,25 @@ async def get_mapped(data: dict, tenant: str = Header(...)): l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + #print("custom attributes: ", custom_attributes) + for attribute in custom_attributes: + preprocess_attribute = re.sub(r'[^a-zA-Z0-9]', '', attribute).lower() + + new_synonym = { + "synonym": preprocess_attribute, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{attribute}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"new synonyms inserted succesfully") + logging.info(f"list 2: {l2}") #print("custom_attributes: ", custom_attributes) @@ -710,7 +731,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- -@app.post("/generativeaisrvc/store_data") +@app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): try: # Check if 'request_id' and 'payload' are present in the request @@ -931,27 +952,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -#------------ API for mapping roles------------- -# @app.post("/generativeaisrvc/map_fields_to_roles") -# async def map_fields_to_roles(payload: Dict[str, Any]): -# try: -# body = payload.get("body") -# policy_mapping = payload.get("policyMapping") - -# if not body: -# raise HTTPException(status_code=400, detail="body empty") -# elif not policy_mapping: -# raise HTTPException(status_code=400, detail="policy_mapping empty") - -# except HTTPException: -# raise -# except Exception as e: -# return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") - - + raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index 929f8d9..5da9274 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -20,6 +20,7 @@ import uvicorn import uuid from typing import Set +import re app = FastAPI() @@ -252,11 +253,12 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): # Check similarity with original list (l2) for element_l2 in l2: el1 = str(element_l1).lower() + el1 = re.sub(r'[^a-zA-Z0-9]', '', el1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio(el1, el2) - if similarity > max_similarity and similarity >= threshold: - max_similarity = similarity - matching_element_l2 = element_l2 + # similarity = fuzz.ratio(el1, el2) + # if similarity > max_similarity and similarity >= threshold: + # max_similarity = similarity + # matching_element_l2 = element_l2 if not matching_element_l2: synonyms_doc = synonyms_collection.find_one() @@ -571,6 +573,25 @@ async def get_mapped(data: dict, tenant: str = Header(...)): l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + #print("custom attributes: ", custom_attributes) + for attribute in custom_attributes: + preprocess_attribute = re.sub(r'[^a-zA-Z0-9]', '', attribute).lower() + + new_synonym = { + "synonym": preprocess_attribute, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{attribute}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"new synonyms inserted succesfully") + logging.info(f"list 2: {l2}") #print("custom_attributes: ", custom_attributes) From 68afa2a20f2f65774fa905327c5431aa4b26a086 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 3 Apr 2024 14:54:48 +0530 Subject: [PATCH 69/91] host changed --- routes.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.yaml b/routes.yaml index 401180f..b622dd9 100644 --- a/routes.yaml +++ b/routes.yaml @@ -42,7 +42,7 @@ spec: type: prefix url: /generativeaisrvc/get_policy_mapped targets: - - host: generativeaiservice.unotech.svc.cluster.local + - host: 10.0.1.4 port: 8080 scheme: http type: "" @@ -93,7 +93,7 @@ spec: type: prefix url: /generativeaisrvc/map_fields_to_policy targets: - - host: generativeaiservice.unotech.svc.cluster.local + - host: 10.0.1.4 port: 8080 scheme: http type: "" @@ -144,7 +144,7 @@ spec: type: prefix url: /generativeaisrvc/feedback targets: - - host: generativeaiservice.unotech.svc.cluster.local + - host: 10.0.1.4 port: 8080 scheme: http type: "" From 78f2545199c59e0c91a10202d127b90fac00c101 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 3 Apr 2024 17:04:46 +0530 Subject: [PATCH 70/91] commeted some unused statement for feedback api --- dev.py | 15 +++++++++------ policy_mapping.py | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dev.py b/dev.py index 5da9274..483ddf6 100644 --- a/dev.py +++ b/dev.py @@ -733,14 +733,16 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): + print("tenant: ",tenant) try: # Check if 'request_id' and 'payload' are present in the request - if 'request_Id' not in payload: - raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") - elif 'payload' not in payload: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - + # if 'request_Id' not in payload: + # raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") + # elif 'payload' not in payload: + # raise HTTPException(status_code=400, detail="Missing 'payload' in request") + logging.debug(f" The payload is {payload}") request_Id = payload.get("request_id") + logging.debug(f" The request_Id is {request_Id}") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) @@ -952,7 +954,8 @@ async def store_data(payload: dict, tenant: str = Header(None)): #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} except Exception as e: + print("faileddddddd") raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index 5da9274..483ddf6 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -733,14 +733,16 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): + print("tenant: ",tenant) try: # Check if 'request_id' and 'payload' are present in the request - if 'request_Id' not in payload: - raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") - elif 'payload' not in payload: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - + # if 'request_Id' not in payload: + # raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") + # elif 'payload' not in payload: + # raise HTTPException(status_code=400, detail="Missing 'payload' in request") + logging.debug(f" The payload is {payload}") request_Id = payload.get("request_id") + logging.debug(f" The request_Id is {request_Id}") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) @@ -952,7 +954,8 @@ async def store_data(payload: dict, tenant: str = Header(None)): #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} except Exception as e: + print("faileddddddd") raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file From 016e54d3ea1bf47930f240e4b46e40d6b6a6e9e1 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 11:36:12 +0530 Subject: [PATCH 71/91] updated routes.yaml with keep rewrite as empty string --- routes.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.yaml b/routes.yaml index b622dd9..029a4aa 100644 --- a/routes.yaml +++ b/routes.yaml @@ -38,7 +38,7 @@ spec: methods: - '*' port: 0 - rewrite: /generativeaisrvc/get_policy_mapped + rewrite: "" type: prefix url: /generativeaisrvc/get_policy_mapped targets: @@ -89,7 +89,7 @@ spec: methods: - '*' port: 0 - rewrite: /generativeaisrvc/map_fields_to_policy + rewrite: "" type: prefix url: /generativeaisrvc/map_fields_to_policy targets: @@ -140,7 +140,7 @@ spec: methods: - '*' port: 0 - rewrite: /generativeaisrvc/feedback + rewrite: "" type: prefix url: /generativeaisrvc/feedback targets: From 3cece30a0f6b00ca3f816f284f1870bff6d7dec4 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 15:21:57 +0530 Subject: [PATCH 72/91] updated logging statements --- dev.py | 48 ++++++++++++++++++++++++++++------------------- policy_mapping.py | 48 ++++++++++++++++++++++++++++------------------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/dev.py b/dev.py index 483ddf6..800713a 100644 --- a/dev.py +++ b/dev.py @@ -733,27 +733,30 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): - print("tenant: ",tenant) + logging.debug(f"API call for feedback given by admin") + + logging.debug(f"working on tenant: {tenant}") try: # Check if 'request_id' and 'payload' are present in the request - # if 'request_Id' not in payload: - # raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") - # elif 'payload' not in payload: - # raise HTTPException(status_code=400, detail="Missing 'payload' in request") + if payload is None: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + elif 'request_id' not in payload: + raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + logging.debug(f" The payload is {payload}") - request_Id = payload.get("request_id") - logging.debug(f" The request_Id is {request_Id}") + request_id = payload.get("request_id") + logging.debug(f" The request_id is {request_id}") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) - logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") + logging.debug(f"Data inserted succesfully for request_id : {request_id}") # query AI suggestion collection subset_collection = stored_policy_mapped(tenant) - doc1 = subset_collection.find_one({"request_id":request_Id}) + doc1 = subset_collection.find_one({"request_id":request_id}) # query admin collection - doc2 = policymap_collection.find_one({"request_id":request_Id}) + doc2 = policymap_collection.find_one({"request_id":request_id}) #query global collection synonyms_collection = get_master_collection("amayaSynonymsMaster") @@ -766,7 +769,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() print("attribute_name of the application: ",attribute_name1) @@ -774,6 +777,9 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Fetching l2_matched from doc1 l2_matched1 = policy1.get("l2_matched") print("l2_matched suggested by AI: ",l2_matched1) + + l2matched2 = policy2.get("l2_matched") + print("l2_matched given by admin",l2matched2) # Finding the attribute in the global collection pipeline = [ @@ -825,7 +831,8 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name1} to {new_score} since the suggestion given was wrong by AI") + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI and it should be {l2_matched2}.") + logging.debug(f"End of calculation logic.") else: logging.debug("No 'synonyms' found in the document.") @@ -851,10 +858,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert=True ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym} for {l2_matched2}") + logging.debug(f"End of calculation logic") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") attribute_name = policy1.get("attributeName").lower() print("attribute_name of the application: ",attribute_name) @@ -918,12 +926,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name} to {new_score} since the suggestion given was right by AI") + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") + else: logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty string") + logging.debug(f" checking and updating where matching decision is empty string.") attribute_name2 = policy2.get("attributeName").lower() print("attribute_name of the application: ",attribute_name2) @@ -945,11 +955,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - logging.debug(f"Inserted new synonym: {new_synonym}") - + logging.debug(f"Inserted new synonym: {new_synonym} for {l2_matched2}.") + logging.debug(f"End of calculation logic.") else: - logging.debug("no need to analyze and changed") + logging.debug("no need to analyze and changed.") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} diff --git a/policy_mapping.py b/policy_mapping.py index 483ddf6..800713a 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -733,27 +733,30 @@ async def map_fields_to_policy(payload: Dict[str, Any]): #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): - print("tenant: ",tenant) + logging.debug(f"API call for feedback given by admin") + + logging.debug(f"working on tenant: {tenant}") try: # Check if 'request_id' and 'payload' are present in the request - # if 'request_Id' not in payload: - # raise HTTPException(status_code=400, detail="Missing 'request_Id' in request") - # elif 'payload' not in payload: - # raise HTTPException(status_code=400, detail="Missing 'payload' in request") + if payload is None: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + elif 'request_id' not in payload: + raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + logging.debug(f" The payload is {payload}") - request_Id = payload.get("request_id") - logging.debug(f" The request_Id is {request_Id}") + request_id = payload.get("request_id") + logging.debug(f" The request_id is {request_id}") policymap_collection = stored_admin_policymap(tenant) policymap_collection.insert_one(payload) - logging.debug(f"Data inserted succesfully for request_Id : {request_Id}") + logging.debug(f"Data inserted succesfully for request_id : {request_id}") # query AI suggestion collection subset_collection = stored_policy_mapped(tenant) - doc1 = subset_collection.find_one({"request_id":request_Id}) + doc1 = subset_collection.find_one({"request_id":request_id}) # query admin collection - doc2 = policymap_collection.find_one({"request_id":request_Id}) + doc2 = policymap_collection.find_one({"request_id":request_id}) #query global collection synonyms_collection = get_master_collection("amayaSynonymsMaster") @@ -766,7 +769,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): # print("policy2: ",policy2) if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() print("attribute_name of the application: ",attribute_name1) @@ -774,6 +777,9 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Fetching l2_matched from doc1 l2_matched1 = policy1.get("l2_matched") print("l2_matched suggested by AI: ",l2_matched1) + + l2matched2 = policy2.get("l2_matched") + print("l2_matched given by admin",l2matched2) # Finding the attribute in the global collection pipeline = [ @@ -825,7 +831,8 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name1} to {new_score} since the suggestion given was wrong by AI") + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI and it should be {l2_matched2}.") + logging.debug(f"End of calculation logic.") else: logging.debug("No 'synonyms' found in the document.") @@ -851,10 +858,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert=True ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym} for {l2_matched2}") + logging.debug(f"End of calculation logic") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal") + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") attribute_name = policy1.get("attributeName").lower() print("attribute_name of the application: ",attribute_name) @@ -918,12 +926,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name} to {new_score} since the suggestion given was right by AI") + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") + else: logging.debug("No 'synonyms' found in the document.") elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty string") + logging.debug(f" checking and updating where matching decision is empty string.") attribute_name2 = policy2.get("attributeName").lower() print("attribute_name of the application: ",attribute_name2) @@ -945,11 +955,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - logging.debug(f"Inserted new synonym: {new_synonym}") - + logging.debug(f"Inserted new synonym: {new_synonym} for {l2_matched2}.") + logging.debug(f"End of calculation logic.") else: - logging.debug("no need to analyze and changed") + logging.debug("no need to analyze and changed.") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} From 3d3b0cfcdbf868ea6411ab86e290166b9fe4ce5c Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 15:56:58 +0530 Subject: [PATCH 73/91] updated file with adding middleName into the list2 for cymmetri fields --- dev.py | 3 ++- policy_mapping.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev.py b/dev.py index 800713a..7a816c4 100644 --- a/dev.py +++ b/dev.py @@ -541,7 +541,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'middleName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { 'department': 'STRING', @@ -555,6 +555,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): 'email': 'STRING', 'end_date': 'DATE', 'firstName': 'STRING', + 'middleName': 'STRING', 'login': 'STRING', 'lastName': 'STRING', 'userType': 'STRING', diff --git a/policy_mapping.py b/policy_mapping.py index 800713a..7a816c4 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -541,7 +541,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'middleName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] l2_datatypes = { 'department': 'STRING', @@ -555,6 +555,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): 'email': 'STRING', 'end_date': 'DATE', 'firstName': 'STRING', + 'middleName': 'STRING', 'login': 'STRING', 'lastName': 'STRING', 'userType': 'STRING', From 51e8cd6f39abd6ec085fff5cced7a7c86f34ec5f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 18:23:09 +0530 Subject: [PATCH 74/91] fix issues with feedback api for l2matched2 variable --- dev.py | 8 +++----- policy_mapping.py | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/dev.py b/dev.py index 7a816c4..0fbc617 100644 --- a/dev.py +++ b/dev.py @@ -763,8 +763,6 @@ async def store_data(payload: dict, tenant: str = Header(None)): synonyms_collection = get_master_collection("amayaSynonymsMaster") if doc1 and doc2: - # print("doc1: ",doc1) - # print("doc2: ",doc2) for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): # print("policy1: ",policy1) # print("policy2: ",policy2) @@ -832,7 +830,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI and it should be {l2_matched2}.") + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") logging.debug(f"End of calculation logic.") else: logging.debug("No 'synonyms' found in the document.") @@ -859,7 +857,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert=True ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym} for {l2_matched2}") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") logging.debug(f"End of calculation logic") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): @@ -956,7 +954,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - logging.debug(f"Inserted new synonym: {new_synonym} for {l2_matched2}.") + logging.debug(f"Inserted new synonym: {new_synonym}.") logging.debug(f"End of calculation logic.") else: diff --git a/policy_mapping.py b/policy_mapping.py index 7a816c4..0fbc617 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -763,8 +763,6 @@ async def store_data(payload: dict, tenant: str = Header(None)): synonyms_collection = get_master_collection("amayaSynonymsMaster") if doc1 and doc2: - # print("doc1: ",doc1) - # print("doc2: ",doc2) for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): # print("policy1: ",policy1) # print("policy2: ",policy2) @@ -832,7 +830,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert= True ) - logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI and it should be {l2_matched2}.") + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") logging.debug(f"End of calculation logic.") else: logging.debug("No 'synonyms' found in the document.") @@ -859,7 +857,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): upsert=True ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym} for {l2_matched2}") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") logging.debug(f"End of calculation logic") elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): @@ -956,7 +954,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): }, upsert=True ) - logging.debug(f"Inserted new synonym: {new_synonym} for {l2_matched2}.") + logging.debug(f"Inserted new synonym: {new_synonym}.") logging.debug(f"End of calculation logic.") else: From 2233d6de2d1c4de54d93aec83cd5276a6a326a3a Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 19:29:46 +0530 Subject: [PATCH 75/91] added error code for api --- backup_dev.py | 712 +++++++++++++++++++++++++++++++--------------- dev.py | 101 +++++-- policy_mapping.py | 101 +++++-- 3 files changed, 648 insertions(+), 266 deletions(-) diff --git a/backup_dev.py b/backup_dev.py index bbdb6a0..0fbc617 100644 --- a/backup_dev.py +++ b/backup_dev.py @@ -20,6 +20,7 @@ import uvicorn import uuid from typing import Set +import re app = FastAPI() @@ -42,33 +43,27 @@ def ErrorResponseModel(error, code, message): #--------- stored the payload as input---------- def stored_input(tenant: str): - return get_collection(tenant, "schema_maker_input") + return get_collection(tenant, "amaya_input") #--------------stored policymap for all users---------------- def stored_response(tenant: str): - return get_collection(tenant, "schema_maker_final_output") + return get_collection(tenant, "amaya_final_output") #------------subset policy map response-------------- def stored_policy_mapped(tenant: str): - return get_collection(tenant, "schema_maker_policyMap") - -#---------synonyms_dict for training-------- -# def retreive_synonyms(tenant: str): -# return get_collection(tenant, "amayaSynonymsMaster") + return get_collection(tenant, "amaya_policyMap") #------final policymap by admin for training purpose--------- def stored_admin_policymap(tenant: str): - return get_collection(tenant, "schema_maker_final_policyMap") + return get_collection(tenant, "amaya_final_policyMap") #----------custom attributes for appending in cymmetri list----- def retrieve_custom_attributes(tenant: str): return get_collection(tenant, "custome_attribute_master") - - #----------- score for confidence level def stored_score(tenant: str, appId: str): - score_collection = get_collection(tenant, "schema_maker_score") + score_collection = get_collection(tenant, "amaya_score") # Check if index exists #index_exists = appId in score_collection.index_information() @@ -96,6 +91,10 @@ def convert_string_to_list(input_str: str) -> List[str]: # Remove leading and trailing whitespaces, and split by ',' return [element.strip() for element in input_str.strip('[]').split(',')] +#--------------for preprocess purpose +def remove_underscores_from_set(input_set): + return set(element.replace('_', '') for element in input_set) + #--------generate request_id----------- def generate_request_id(): id = uuid.uuid1() @@ -130,31 +129,36 @@ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): #-----------------------------extracting the user object from response----------------- def extract_user_data(response): try: - logging.debug(f"extracting the users from the nested json") + logging.debug("Extracting the users from the nested json") user_data_list = [] def is_user_data(obj): - # Check if object contains at least one of the common user data keys - user_keys = {'displayName', 'givenName' 'email', 'id', 'DateOfBirth'} - return any(key in obj for key in user_keys) - - def traverse(obj): + # Convert keys in the object to lowercase for case-insensitive comparison + lower_case_obj_keys = {key.lower() for key in obj.keys()} + # Define user data keys in lowercase + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + # Check if object contains at least one of the common user data keys, ignoring case + return any(key in lower_case_obj_keys for key in user_keys) + + def traverse(obj, original_keys=None): # Recursively traverse the JSON object - nonlocal user_data_list if isinstance(obj, dict): if is_user_data(obj): - user_data_list.append(obj) + # Convert keys in the user data to lowercase or handle as needed + user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) else: - for value in obj.values(): - traverse(value) + for key, value in obj.items(): + # Maintain original keys for nested dictionaries + traverse(value, {**original_keys, **{key.lower(): key}}) elif isinstance(obj, list): for item in obj: - traverse(item) + traverse(item, original_keys) - traverse(response) + traverse(response, {}) return user_data_list except Exception as e: + logging.error(f"Error extracting user data: {e}") raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") #---------------------------extracting keys, datatype, label and jsonpath---------------- @@ -172,11 +176,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - "label": key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -230,7 +235,7 @@ def get_data_type(value): return distinct_keys_datatypes except Exception as e: raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") - + #-------------------fuzzy logic matching function---------------------- def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): @@ -238,7 +243,8 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): matching_elements_l2 = [] non_matching_elements_l1 = [] non_matching_elements_l2 = [] - + similar_elements = [] + for element_l1 in l1: max_similarity = 0 matching_element_l2 = '' @@ -247,47 +253,30 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): # Check similarity with original list (l2) for element_l2 in l2: el1 = str(element_l1).lower() + el1 = re.sub(r'[^a-zA-Z0-9]', '', el1).lower() el2 = str(element_l2).lower() - similarity = fuzz.ratio(el1, el2) - if similarity > max_similarity and similarity >= threshold: - max_similarity = similarity - matching_element_l2 = element_l2 - - # If no match is found in the original list, check synonyms - # if not matching_element_l2: - # synonyms_doc = synonyms_collection.find_one() - # if synonyms_doc: - # synonyms = synonyms_doc.get("synonyms", {}) - # for key, values in synonyms.items(): - # for synonym_obj in values: - # synonym = synonym_obj.get("synonym", "").lower() - # score = synonym_obj.get("score", 0) - # adjusted_score = score * 100 - # if el1 == synonym and adjusted_score >= threshold: - # matching_element_l2 = key - # max_similarity = adjusted_score # Use score as similarity percentage - # is_synonym_match = True - # break - # if is_synonym_match: - # break + # similarity = fuzz.ratio(el1, el2) + # if similarity > max_similarity and similarity >= threshold: + # max_similarity = similarity + # matching_element_l2 = element_l2 if not matching_element_l2: synonyms_doc = synonyms_collection.find_one() if synonyms_doc: - threshold_synonyms = threshold / 100 + threshold_synonyms = threshold / 100 data = synonyms_collection.aggregate([ { "$project": { - "synonymsArray": { "$objectToArray": "$synonyms" } + "synonymsArray": {"$objectToArray": "$synonyms"} } }, - { "$unwind": "$synonymsArray" }, + {"$unwind": "$synonymsArray"}, { "$match": { "synonymsArray.v": { "$elemMatch": { "synonym": el1, - "score": { "$gt": threshold_synonyms } + "score": {"$gt": threshold_synonyms} } } } @@ -302,63 +291,64 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): "as": "syn", "cond": { "$and": [ - { "$eq": ["$$syn.synonym", el1] }, - { "$gt": ["$$syn.score", threshold_synonyms] } + {"$eq": ["$$syn.synonym", el1]}, + {"$gt": ["$$syn.score", threshold_synonyms]} ] } } } } }, - { "$limit": 1 } # Limiting to one document + {"$limit": 1} ]) if data: - for document in data: - matching_element_l2 = document.get('key', None) - synonyms = document.get('synonyms', []) - max_similarity = None - if synonyms: - max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) - is_synonym_match = True - break - + for document in data: + matching_element_l2 = document.get('key', None) + synonyms = document.get('synonyms', []) + max_similarity = None + if synonyms: + max_similarity = max(synonyms, key=lambda x: x.get('score', 0)).get('score', None) + is_synonym_match = True + break + else: matching_element_l2 = None max_similarity = None - if matching_element_l2: matching_elements_l1.append(element_l1.strip("'")) matching_elements_l2.append(matching_element_l2.strip("'")) if is_synonym_match: - print(f"Match found for '{element_l1}' with synonym '{matching_element_l2}' (Score: {max_similarity})") + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": max_similarity, + "matching_decision": "synonyms" + }) else: - print(f"Match found for '{element_l1}' with fuzzy '{matching_element_l2}' (Similarity: {max_similarity})") + similarity_percentage = fuzz.ratio(element_l1.lower(), matching_element_l2.lower()) / 100 + similar_elements.append({ + "element_name_l1": element_l1, + "element_name_l2": matching_element_l2, + "similarity_percentage": similarity_percentage, + "matching_decision": "fuzzy" + }) else: non_matching_elements_l1.append(element_l1.strip("'")) - + non_matching_elements_l2 = [ element_l2.strip("'") for element_l2 in l2 if element_l2.strip("'") not in matching_elements_l2 ] - - similar_elements = [] - for element_l1, element_l2 in zip(matching_elements_l1, matching_elements_l2): - similarity_percentage = fuzz.ratio(element_l1.lower(), element_l2.lower()) / 100 - matching_condition = "Fuzzy" - similar_elements.append({ - "element_name_l1": element_l1, - "element_name_l2": element_l2, - "similarity_percentage": similarity_percentage, - "matching_condition": matching_condition # Adding matching condition to the result - }) - - result = {"similar_elements": similar_elements} + result = { + "similar_elements": similar_elements + } return result + #----------------------to get the confidence level based on schema_maker_score def get_confidence_level(similarity_score: float, score_collection) -> str: try: @@ -418,12 +408,13 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo 'value': match['value'], 'similarity_percentage': element['similarity_percentage'], 'confidence': confidence, - 'matching_condition': element["matching_condition"], + 'matching_decision': element["matching_decision"], 'isCustom': is_custom }) processed_labels.add(element['element_name_l1']) else: - print(f"No matched data found for {element['element_name_l1']}") + logging.debug(f"No matched data found for {element['element_name_l1']}") + for data in response_data: if data['label'] not in processed_labels: @@ -437,7 +428,7 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo 'value': data['value'], 'similarity_percentage': 0, 'confidence': confidence, - 'matching_condition': "", + 'matching_decision': "", 'isCustom': False # Explicitly false since there's no l2 match }) @@ -453,7 +444,7 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str internal_field = map_entry["internal"] if external_field.lower() == field.lower(): matched = True - print(f"Exact match found: '{field}' -> '{external_field}'") + logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found @@ -462,12 +453,13 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str for map_entry in policy_mapping: if map_entry["internal"].lower() == best_match: matched = True - print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: - print(f"No match found for '{field}'") - return field, None # Return original field if no match is found + logging.debug(f"No match found for '{field}'") + return field, None + #------- works on nested conditions also def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: @@ -489,7 +481,6 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): - print("Headers:", tenant) logging.debug(f"API call for auto policy mapping with the application") try: @@ -513,159 +504,191 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Validate the format of 'payload' if not isinstance(data['payload'], dict): - raise HTTPException(status_code=400, detail="'payload' must be a dictionary") - + if isinstance(data['payload'], list): + # Convert list of dictionaries to a single dictionary + converted_payload = {} + for item in data['payload']: + for key, value in item.items(): + converted_payload[key] = value + data['payload'] = converted_payload + else: + raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") json_data = data.get('payload') #print("json data is {}", json_data) - #End of changes by Abhishek json_data_ = extract_user_data(json_data) - #print("json_data: ",json_data_) - response_data = get_distinct_keys_and_datatypes(json_data_) - #response_data=list(response_data.values()) - #print("response_data:", response_data) + if json_data_: + logging.info(f" Successfully extract the json data from response") + response_data = get_distinct_keys_and_datatypes(json_data_) + #response_data=list(response_data.values()) + #print("response_data:", response_data) - l1 = [item['label'] for item in response_data] - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) - else: - l1_list = set(l1) - print("list1: ",l1_list) - - - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] - - l2_datatypes = { - 'department': 'STRING', - 'employeeId': 'STRING', - 'designation': 'STRING', - 'appUpdatedDate': 'DATETIME', - 'displayName': 'STRING', - 'mobile': 'STRING', - 'country': 'STRING', - 'city': 'STRING', - 'email': 'STRING', - 'end_date': 'DATE', - 'firstName': 'STRING', - 'login': 'STRING', - 'lastName': 'STRING', - 'userType': 'STRING', - 'end_date': 'DATE', - 'login': 'STRING ', - 'userType': 'STRING', - 'dateOfBirth': 'DATE', - 'endDate': 'DATE', - 'startDate': 'DATE', - 'password': 'password', - 'status': 'STRING', - 'profilePicture': 'profilePicture', - 'appUserId': 'STRING', - 'landline': 'STRING' - } - - l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + l1 = [item['label'] for item in response_data] - print("list2: ",l2) + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + logging.info(f"list1: {l1_list}") + else: + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) + logging.info(f"list1: {l1_list}") + + + + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'middleName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + + l2_datatypes = { + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'middleName': 'STRING', + 'login': 'STRING', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'STRING', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' + } + + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) - #print("custom_attributes: ", custom_attributes) - - if isinstance(l2, str): - l2_list = convert_string_to_list(l2) - else: - l2_list = l2 + #print("custom attributes: ", custom_attributes) + for attribute in custom_attributes: + preprocess_attribute = re.sub(r'[^a-zA-Z0-9]', '', attribute).lower() - threshold = 60 - appId = data.get("appId") + new_synonym = { + "synonym": preprocess_attribute, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{attribute}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"new synonyms inserted succesfully") - #--- Inserting the synonyms_dict for appId-------- - #synonyms_stored_collection = stored_synonyms_dict(tenant, appId, synonyms_dict) + logging.info(f"list 2: {l2}") - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) - - request_id = generate_request_id() - - score_collection = stored_score(tenant, appId) - - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) - #print("final response: ",final_response) - final_response_dict = {"final_response": final_response} - - # Assuming 'appId' is present in the received response - final_response_dict['appId'] = appId - - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) - - logging.debug("Final response saved successfully") - - subset_response = output_collection.aggregate([ - {"$unwind": "$final_response"}, - {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, - {"$group": { - "_id": "$final_response.attributeName", - "data": {"$first": "$final_response"} - }}, - {"$project": { - "_id": 0, - "jsonPath": "$data.jsonPath", - "attributeName": "$data.attributeName", - "l1_datatype": "$data.l1_datatype", - "l2_matched": "$data.l2_matched", - "l2_datatype": "$data.l2_datatype", - "value": "$data.value", - "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence", - "matching_condition": "$data.matching_condition", - "isCustom": "$data.isCustom" - }} - ]) - - - subset_response_data = list(subset_response) - # Serialize each document into a JSON serializable format - json_serializable_response = [] - for doc in subset_response_data: - json_serializable_doc = { - "jsonPath": doc["jsonPath"], - "attributeName": doc["attributeName"], - "l1_datatype": doc["l1_datatype"], - "l2_matched": doc["l2_matched"], - "l2_datatype": doc["l2_datatype"], - "value": doc["value"], - "similarity_percentage": doc["similarity_percentage"], - "confidence": doc["confidence"], - "matching_condition": doc["matching_condition"], - "isCustom": doc["isCustom"] - } - json_serializable_response.append(json_serializable_doc) + #print("custom_attributes: ", custom_attributes) + + if isinstance(l2, str): + l2_list = convert_string_to_list(l2) + else: + l2_list = l2 + threshold = 60 + appId = data.get("appId") - aggregated_data = { - "appId": appId, - "request_id": request_id, - "policyMapList": subset_response_data - } + result = compare_lists_with_fuzzy(l1_list, l2_list, threshold, synonyms_collection) + #print("result: ", result) + + request_id = generate_request_id() + + score_collection = stored_score(tenant, appId) + + final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes, score_collection, custom_attributes) + #print("final response: ",final_response) + final_response_dict = {"final_response": final_response} + + # Assuming 'appId' is present in the received response + final_response_dict['appId'] = appId + + output_collection.update_one( + {"appId": appId}, + {"$set": final_response_dict}, + upsert=True + ) + + logging.debug("Final response saved successfully") + + subset_response = output_collection.aggregate([ + {"$unwind": "$final_response"}, + {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, + {"$group": { + "_id": "$final_response.attributeName", + "data": {"$first": "$final_response"} + }}, + {"$project": { + "_id": 0, + "jsonPath": "$data.jsonPath", + "attributeName": "$data.attributeName", + "l1_datatype": "$data.l1_datatype", + "l2_matched": "$data.l2_matched", + "l2_datatype": "$data.l2_datatype", + "value": "$data.value", + "similarity_percentage": "$data.similarity_percentage", + "confidence": "$data.confidence", + "matching_decision": "$data.matching_decision", + "isCustom": "$data.isCustom" + }} + ]) + + + subset_response_data = list(subset_response) + # Serialize each document into a JSON serializable format + json_serializable_response = [] + for doc in subset_response_data: + json_serializable_doc = { + "jsonPath": doc["jsonPath"], + "attributeName": doc["attributeName"], + "l1_datatype": doc["l1_datatype"], + "l2_matched": doc["l2_matched"], + "l2_datatype": doc["l2_datatype"], + "value": doc["value"], + "similarity_percentage": doc["similarity_percentage"], + "confidence": doc["confidence"], + "matching_decision": doc["matching_decision"], + "isCustom": doc["isCustom"] + } + json_serializable_response.append(json_serializable_doc) + + + aggregated_data = { + "appId": appId, + "request_id": request_id, + "policyMapList": subset_response_data + } - subset_collection.insert_one(aggregated_data) + subset_collection.insert_one(aggregated_data) - logging.debug("subset response saved successfully") + logging.debug("subset response saved successfully") - data_response = { - "request_id": request_id, - "content": json_serializable_response - } + data_response = { + "request_id": request_id, + "content": json_serializable_response + } - #return JSONResponse(content=json_serializable_response) - return ResponseModel(data=data_response, message="Policy mapping generated successfully") + #return JSONResponse(content=json_serializable_response) + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + + else: + logging.info(f" Failed to extract the data from the response") except HTTPException: raise @@ -674,7 +697,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") #------- Api for body populating---------- -@app.post("/generativeaisrvc/map_fields_to_policy/") +@app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): try: body = payload.get("body") @@ -707,16 +730,241 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + #-------------------Api fpr storing the admin final policymap for training purpose----------- -@app.post("/generativeaisrvc/store_data") +@app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): + logging.debug(f"API call for feedback given by admin") + + logging.debug(f"working on tenant: {tenant}") try: - policymap_colection = stored_admin_policymap(tenant) - policymap_colection.insert_one(payload) + # Check if 'request_id' and 'payload' are present in the request + if payload is None: + raise HTTPException(status_code=400, detail="Missing 'payload' in request") + elif 'request_id' not in payload: + raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + + logging.debug(f" The payload is {payload}") + request_id = payload.get("request_id") + logging.debug(f" The request_id is {request_id}") + policymap_collection = stored_admin_policymap(tenant) + policymap_collection.insert_one(payload) + + logging.debug(f"Data inserted succesfully for request_id : {request_id}") + + # query AI suggestion collection + subset_collection = stored_policy_mapped(tenant) + doc1 = subset_collection.find_one({"request_id":request_id}) + + # query admin collection + doc2 = policymap_collection.find_one({"request_id":request_id}) + + #query global collection + synonyms_collection = get_master_collection("amayaSynonymsMaster") + + if doc1 and doc2: + for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): + # print("policy1: ",policy1) + # print("policy2: ",policy2) + + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") + #Fetching attributeName from doc1 + attribute_name1 = policy1.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name1) + + # Fetching l2_matched from doc1 + l2_matched1 = policy1.get("l2_matched") + print("l2_matched suggested by AI: ",l2_matched1) + + l2matched2 = policy2.get("l2_matched") + print("l2_matched given by admin",l2matched2) + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched1}.synonym": attribute_name1 + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched1}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name1] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name1) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") + logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") + + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name2) + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched by admin: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"End of calculation logic") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") + attribute_name = policy1.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name) + + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + print("l2_matched suggested by AI: ", l2_matched) + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } + } + } + } + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + + synonyms = global_doc.get("synonyms", {}) + if synonyms: + + score = global_doc['synonyms']['score'] + + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) + + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") + + else: + logging.debug("No 'synonyms' found in the document.") + + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty string.") + + attribute_name2 = policy2.get("attributeName").lower() + print("attribute_name of the application: ",attribute_name2) + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + print("l2_matched by admin: ",l2_matched2) + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}.") + logging.debug(f"End of calculation logic.") + + else: + logging.debug("no need to analyze and changed.") + + #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - + print("faileddddddd") + raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file diff --git a/dev.py b/dev.py index 0fbc617..75c3c0f 100644 --- a/dev.py +++ b/dev.py @@ -32,14 +32,21 @@ def ResponseModel(data, message, code=200, error_code=None): return { + "success": True, "data": data, "code": code, "message": message, "error_code": error_code } -def ErrorResponseModel(error, code, message): - return {"error": error, "code": code, "message": message} +def ErrorResponseModel(data, success, code, message, error_code): + return { + "data": None, + "success": False, + "code": code, + "message": message, + "error_code": error_code + } #--------- stored the payload as input---------- def stored_input(tenant: str): @@ -375,6 +382,7 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: return "Unknown" # No matching range found, return Unknown except Exception as e: raise HTTPException(status_code=400, detail="score_collection_error") + #----------------------generates final response--------------- @@ -498,10 +506,23 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - raise HTTPException(status_code=400, detail="Missing 'appId' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'appId' in request", + error_code="APPID_MISSING_ERROR" + ) + elif 'payload' not in data: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'payload' in request", + error_code="PAYLOAD_MISSING_ERROR" + ) + # Validate the format of 'payload' if not isinstance(data['payload'], dict): if isinstance(data['payload'], list): @@ -512,7 +533,14 @@ async def get_mapped(data: dict, tenant: str = Header(...)): converted_payload[key] = value data['payload'] = converted_payload else: - raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") + #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="payload' must be a dictionary or list", + error_code="MUST_BE_DICT_OR_LIST" + ) json_data = data.get('payload') @@ -704,9 +732,24 @@ async def map_fields_to_policy(payload: Dict[str, Any]): policy_mapping = payload.get("policyMapping") if not body: - raise HTTPException(status_code=400, detail="body empty") + #raise HTTPException(status_code=400, detail="body empty") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'body' in request", + error_code="BODY_MISSING_ERROR" + ) elif not policy_mapping: - raise HTTPException(status_code=400, detail="policy_mapping empty") + #raise HTTPException(status_code=400, detail="policy_mapping empty") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'policy_mapping' in request", + error_code="POLICY_MAPPING_MISSING_ERROR" + ) + mapped_data = {} @@ -723,14 +766,15 @@ async def map_fields_to_policy(payload: Dict[str, Any]): mapped_data[field] = value return mapped_data + #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") + except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -738,11 +782,29 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"working on tenant: {tenant}") try: - # Check if 'request_id' and 'payload' are present in the request - if payload is None: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - elif 'request_id' not in payload: - raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + request_id = payload.get("request_id") + policyMapList = payload.get("policyMapList") + + # Check if 'request_id' and 'policyMapList' are present in the request + if not policyMapList : + #raise HTTPException(status_code=400, detail="Missing 'payload' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'policyMapList' in request", + error_code="POLICYMAPLIST_MISSING" + ) + elif not request_id : + #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'request_id' in request", + error_code="REQUEST_ID_MISSING" + ) + logging.debug(f" The payload is {payload}") request_id = payload.get("request_id") @@ -962,9 +1024,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} + except HTTPException: + raise + # except Exception as e: + # print("faileddddddd") except Exception as e: - print("faileddddddd") - raise HTTPException(status_code=500, detail=str(e)) + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.") + #raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index 0fbc617..75c3c0f 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -32,14 +32,21 @@ def ResponseModel(data, message, code=200, error_code=None): return { + "success": True, "data": data, "code": code, "message": message, "error_code": error_code } -def ErrorResponseModel(error, code, message): - return {"error": error, "code": code, "message": message} +def ErrorResponseModel(data, success, code, message, error_code): + return { + "data": None, + "success": False, + "code": code, + "message": message, + "error_code": error_code + } #--------- stored the payload as input---------- def stored_input(tenant: str): @@ -375,6 +382,7 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: return "Unknown" # No matching range found, return Unknown except Exception as e: raise HTTPException(status_code=400, detail="score_collection_error") + #----------------------generates final response--------------- @@ -498,10 +506,23 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - raise HTTPException(status_code=400, detail="Missing 'appId' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'appId' in request", + error_code="APPID_MISSING_ERROR" + ) + elif 'payload' not in data: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'payload' in request", + error_code="PAYLOAD_MISSING_ERROR" + ) + # Validate the format of 'payload' if not isinstance(data['payload'], dict): if isinstance(data['payload'], list): @@ -512,7 +533,14 @@ async def get_mapped(data: dict, tenant: str = Header(...)): converted_payload[key] = value data['payload'] = converted_payload else: - raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") + #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="payload' must be a dictionary or list", + error_code="MUST_BE_DICT_OR_LIST" + ) json_data = data.get('payload') @@ -704,9 +732,24 @@ async def map_fields_to_policy(payload: Dict[str, Any]): policy_mapping = payload.get("policyMapping") if not body: - raise HTTPException(status_code=400, detail="body empty") + #raise HTTPException(status_code=400, detail="body empty") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'body' in request", + error_code="BODY_MISSING_ERROR" + ) elif not policy_mapping: - raise HTTPException(status_code=400, detail="policy_mapping empty") + #raise HTTPException(status_code=400, detail="policy_mapping empty") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'policy_mapping' in request", + error_code="POLICY_MAPPING_MISSING_ERROR" + ) + mapped_data = {} @@ -723,14 +766,15 @@ async def map_fields_to_policy(payload: Dict[str, Any]): mapped_data[field] = value return mapped_data + #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") + except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -738,11 +782,29 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"working on tenant: {tenant}") try: - # Check if 'request_id' and 'payload' are present in the request - if payload is None: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - elif 'request_id' not in payload: - raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + request_id = payload.get("request_id") + policyMapList = payload.get("policyMapList") + + # Check if 'request_id' and 'policyMapList' are present in the request + if not policyMapList : + #raise HTTPException(status_code=400, detail="Missing 'payload' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'policyMapList' in request", + error_code="POLICYMAPLIST_MISSING" + ) + elif not request_id : + #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + return ErrorResponseModel( + data = None, + success= False, + code=400, + message="Missing 'request_id' in request", + error_code="REQUEST_ID_MISSING" + ) + logging.debug(f" The payload is {payload}") request_id = payload.get("request_id") @@ -962,9 +1024,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} + except HTTPException: + raise + # except Exception as e: + # print("faileddddddd") except Exception as e: - print("faileddddddd") - raise HTTPException(status_code=500, detail=str(e)) + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.") + #raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file From 568df32343b26356e17ff5558e9d87b268ce241f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 4 Apr 2024 20:12:59 +0530 Subject: [PATCH 76/91] updated logging statements for debugging --- dev.py | 18 +++++++++--------- policy_mapping.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dev.py b/dev.py index 75c3c0f..1629d3f 100644 --- a/dev.py +++ b/dev.py @@ -833,14 +833,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name1) + logging.debug(f"attribute_name of the application {attribute_name1}") # Fetching l2_matched from doc1 l2_matched1 = policy1.get("l2_matched") - print("l2_matched suggested by AI: ",l2_matched1) + logging.debug(f"l2_matched suggested by AI {l2_matched1}") l2matched2 = policy2.get("l2_matched") - print("l2_matched given by admin",l2matched2) + logging.debug(f"l2_matched given by admin {l2matched2}") # Finding the attribute in the global collection pipeline = [ @@ -899,11 +899,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): #----------------------for storing new synonyms against the admin l2matched--------------------- attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) + logging.debug(f"attribute_name of the application {attribute_name2}") # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) + logging.debug(f"l2_matched by admin {l2_matched2}") new_synonym = { "synonym": attribute_name2, @@ -925,11 +925,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") attribute_name = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name) + logging.debug(f"attribute_name of the application {attribute_name}") # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched suggested by AI: ", l2_matched) + logging.debug(f"l2_matched suggested by AI {l2_matched}") # Finding the attribute in the global collection pipeline = [ @@ -997,11 +997,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f" checking and updating where matching decision is empty string.") attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) + logging.debug(f"attribute_name of the application {attribute_name2}") # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) + logging.debug(f"l2_matched by admin {l2_matched2}") new_synonym = { "synonym": attribute_name2, diff --git a/policy_mapping.py b/policy_mapping.py index 75c3c0f..1629d3f 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -833,14 +833,14 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name1) + logging.debug(f"attribute_name of the application {attribute_name1}") # Fetching l2_matched from doc1 l2_matched1 = policy1.get("l2_matched") - print("l2_matched suggested by AI: ",l2_matched1) + logging.debug(f"l2_matched suggested by AI {l2_matched1}") l2matched2 = policy2.get("l2_matched") - print("l2_matched given by admin",l2matched2) + logging.debug(f"l2_matched given by admin {l2matched2}") # Finding the attribute in the global collection pipeline = [ @@ -899,11 +899,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): #----------------------for storing new synonyms against the admin l2matched--------------------- attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) + logging.debug(f"attribute_name of the application {attribute_name2}") # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) + logging.debug(f"l2_matched by admin {l2_matched2}") new_synonym = { "synonym": attribute_name2, @@ -925,11 +925,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") attribute_name = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name) + logging.debug(f"attribute_name of the application {attribute_name}") # Fetching l2_matched from doc1 l2_matched = policy1.get("l2_matched") - print("l2_matched suggested by AI: ", l2_matched) + logging.debug(f"l2_matched suggested by AI {l2_matched}") # Finding the attribute in the global collection pipeline = [ @@ -997,11 +997,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f" checking and updating where matching decision is empty string.") attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) + logging.debug(f"attribute_name of the application {attribute_name2}") # Fetching l2_matched from doc2 l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) + logging.debug(f"l2_matched by admin {l2_matched2}") new_synonym = { "synonym": attribute_name2, From 9660e932e566b7402bc79a6f7ee19d6725820f85 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 5 Apr 2024 12:15:02 +0530 Subject: [PATCH 77/91] replaced the error response model for issue with status code --- dev.py | 105 ++++++++++++++++++++++------------------------ policy_mapping.py | 105 ++++++++++++++++++++++------------------------ 2 files changed, 98 insertions(+), 112 deletions(-) diff --git a/dev.py b/dev.py index 1629d3f..fa9b842 100644 --- a/dev.py +++ b/dev.py @@ -30,22 +30,22 @@ ) -def ResponseModel(data, message, code=200, error_code=None): +def ResponseModel(data, message, code=200, errorCode=None): return { "success": True, "data": data, "code": code, "message": message, - "error_code": error_code + "errorCode": errorCode } -def ErrorResponseModel(data, success, code, message, error_code): +def ErrorResponseModel(data, success, code, message, errorCode): return { "data": None, "success": False, "code": code, "message": message, - "error_code": error_code + "errorCode": errorCode } #--------- stored the payload as input---------- @@ -506,22 +506,20 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'appId' in request", - error_code="APPID_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "APPID_MISSING_ERROR", + response_val["message"] = "Missing 'appId' in request" + raise HTTPException(status_code=400, detail=response_val) elif 'payload' not in data: - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'payload' in request", - error_code="PAYLOAD_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "PAYLOAD_MISSING_ERROR", + response_val["message"] = "Missing 'payload' in request" + raise HTTPException(status_code=400, detail=response_val) # Validate the format of 'payload' if not isinstance(data['payload'], dict): @@ -534,13 +532,12 @@ async def get_mapped(data: dict, tenant: str = Header(...)): data['payload'] = converted_payload else: #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="payload' must be a dictionary or list", - error_code="MUST_BE_DICT_OR_LIST" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "MUST_BE_DICT_OR_LIST", + response_val["message"] = "payload' must be a dictionary or list" + raise HTTPException(status_code=400, detail=response_val) json_data = data.get('payload') @@ -733,23 +730,22 @@ async def map_fields_to_policy(payload: Dict[str, Any]): if not body: #raise HTTPException(status_code=400, detail="body empty") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'body' in request", - error_code="BODY_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "BODY_MISSING_ERROR", + response_val["message"] = "Missing 'body' in request" + raise HTTPException(status_code=400, detail=response_val) + elif not policy_mapping: #raise HTTPException(status_code=400, detail="policy_mapping empty") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'policy_mapping' in request", - error_code="POLICY_MAPPING_MISSING_ERROR" - ) - + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["code"] = 400, + response_val["errorCode"] = "POLICY_MAPPING_MISSING_ERROR", + response_val["message"] = "Missing 'policy_mapping' in request" + raise HTTPException(status_code=400, detail=response_val) mapped_data = {} @@ -768,13 +764,11 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return mapped_data #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") - except HTTPException: raise except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -788,22 +782,21 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Check if 'request_id' and 'policyMapList' are present in the request if not policyMapList : #raise HTTPException(status_code=400, detail="Missing 'payload' in request") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'policyMapList' in request", - error_code="POLICYMAPLIST_MISSING" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "POLICYMAPLIST_MISSING", + response_val["message"] = "Missing 'policyMapList' in request" + raise HTTPException(status_code=400, detail=response_val) + elif not request_id : #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'request_id' in request", - error_code="REQUEST_ID_MISSING" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "REQUEST_ID_MISSING", + response_val["message"] = "Missing 'request_id' in request" + raise HTTPException(status_code=400, detail=response_val) logging.debug(f" The payload is {payload}") diff --git a/policy_mapping.py b/policy_mapping.py index 1629d3f..fa9b842 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -30,22 +30,22 @@ ) -def ResponseModel(data, message, code=200, error_code=None): +def ResponseModel(data, message, code=200, errorCode=None): return { "success": True, "data": data, "code": code, "message": message, - "error_code": error_code + "errorCode": errorCode } -def ErrorResponseModel(data, success, code, message, error_code): +def ErrorResponseModel(data, success, code, message, errorCode): return { "data": None, "success": False, "code": code, "message": message, - "error_code": error_code + "errorCode": errorCode } #--------- stored the payload as input---------- @@ -506,22 +506,20 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'appId' in request", - error_code="APPID_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "APPID_MISSING_ERROR", + response_val["message"] = "Missing 'appId' in request" + raise HTTPException(status_code=400, detail=response_val) elif 'payload' not in data: - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'payload' in request", - error_code="PAYLOAD_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "PAYLOAD_MISSING_ERROR", + response_val["message"] = "Missing 'payload' in request" + raise HTTPException(status_code=400, detail=response_val) # Validate the format of 'payload' if not isinstance(data['payload'], dict): @@ -534,13 +532,12 @@ async def get_mapped(data: dict, tenant: str = Header(...)): data['payload'] = converted_payload else: #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="payload' must be a dictionary or list", - error_code="MUST_BE_DICT_OR_LIST" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "MUST_BE_DICT_OR_LIST", + response_val["message"] = "payload' must be a dictionary or list" + raise HTTPException(status_code=400, detail=response_val) json_data = data.get('payload') @@ -733,23 +730,22 @@ async def map_fields_to_policy(payload: Dict[str, Any]): if not body: #raise HTTPException(status_code=400, detail="body empty") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'body' in request", - error_code="BODY_MISSING_ERROR" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "BODY_MISSING_ERROR", + response_val["message"] = "Missing 'body' in request" + raise HTTPException(status_code=400, detail=response_val) + elif not policy_mapping: #raise HTTPException(status_code=400, detail="policy_mapping empty") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'policy_mapping' in request", - error_code="POLICY_MAPPING_MISSING_ERROR" - ) - + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["code"] = 400, + response_val["errorCode"] = "POLICY_MAPPING_MISSING_ERROR", + response_val["message"] = "Missing 'policy_mapping' in request" + raise HTTPException(status_code=400, detail=response_val) mapped_data = {} @@ -768,13 +764,11 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return mapped_data #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") - except HTTPException: raise except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -788,22 +782,21 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Check if 'request_id' and 'policyMapList' are present in the request if not policyMapList : #raise HTTPException(status_code=400, detail="Missing 'payload' in request") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'policyMapList' in request", - error_code="POLICYMAPLIST_MISSING" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "POLICYMAPLIST_MISSING", + response_val["message"] = "Missing 'policyMapList' in request" + raise HTTPException(status_code=400, detail=response_val) + elif not request_id : #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") - return ErrorResponseModel( - data = None, - success= False, - code=400, - message="Missing 'request_id' in request", - error_code="REQUEST_ID_MISSING" - ) + response_val ={} + response_val["data"] = None + response_val["success"] = "false", + response_val["errorCode"] = "REQUEST_ID_MISSING", + response_val["message"] = "Missing 'request_id' in request" + raise HTTPException(status_code=400, detail=response_val) logging.debug(f" The payload is {payload}") From 0752e349dd9de18f42d3f1f1e779bb756ba93f5d Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 5 Apr 2024 13:25:14 +0530 Subject: [PATCH 78/91] parameter fixes with errorresponse model --- dev.py | 39 ++++++++++++++++++++------------------- policy_mapping.py | 39 ++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/dev.py b/dev.py index fa9b842..2cbfb88 100644 --- a/dev.py +++ b/dev.py @@ -39,13 +39,14 @@ def ResponseModel(data, message, code=200, errorCode=None): "errorCode": errorCode } -def ErrorResponseModel(data, success, code, message, errorCode): +def ErrorResponseModel(error, code, message, errorCode): return { "data": None, "success": False, "code": code, "message": message, - "errorCode": errorCode + "errorCode": errorCode, + "error": error } #--------- stored the payload as input---------- @@ -729,23 +730,22 @@ async def map_fields_to_policy(payload: Dict[str, Any]): policy_mapping = payload.get("policyMapping") if not body: - #raise HTTPException(status_code=400, detail="body empty") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "BODY_MISSING_ERROR", - response_val["message"] = "Missing 'body' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "BODY_MISSING_ERROR", + "message": "Missing 'body' in request" + } + raise HTTPException(400, detail=response_val) elif not policy_mapping: - #raise HTTPException(status_code=400, detail="policy_mapping empty") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["code"] = 400, - response_val["errorCode"] = "POLICY_MAPPING_MISSING_ERROR", - response_val["message"] = "Missing 'policy_mapping' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "POLICY_MAPPING_MISSING_ERROR", + "message": "Missing 'policy_mapping' in request" + } + raise HTTPException(400, detail=response_val) mapped_data = {} @@ -767,8 +767,9 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") + + #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): diff --git a/policy_mapping.py b/policy_mapping.py index fa9b842..2cbfb88 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -39,13 +39,14 @@ def ResponseModel(data, message, code=200, errorCode=None): "errorCode": errorCode } -def ErrorResponseModel(data, success, code, message, errorCode): +def ErrorResponseModel(error, code, message, errorCode): return { "data": None, "success": False, "code": code, "message": message, - "errorCode": errorCode + "errorCode": errorCode, + "error": error } #--------- stored the payload as input---------- @@ -729,23 +730,22 @@ async def map_fields_to_policy(payload: Dict[str, Any]): policy_mapping = payload.get("policyMapping") if not body: - #raise HTTPException(status_code=400, detail="body empty") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "BODY_MISSING_ERROR", - response_val["message"] = "Missing 'body' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "BODY_MISSING_ERROR", + "message": "Missing 'body' in request" + } + raise HTTPException(400, detail=response_val) elif not policy_mapping: - #raise HTTPException(status_code=400, detail="policy_mapping empty") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["code"] = 400, - response_val["errorCode"] = "POLICY_MAPPING_MISSING_ERROR", - response_val["message"] = "Missing 'policy_mapping' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "POLICY_MAPPING_MISSING_ERROR", + "message": "Missing 'policy_mapping' in request" + } + raise HTTPException(400, detail=response_val) mapped_data = {} @@ -767,8 +767,9 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") + + #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): From dc2e67e3f6d1213d2318e06d0d6c38acf3151c7d Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 5 Apr 2024 16:59:03 +0530 Subject: [PATCH 79/91] created new function for badrequest --- dev.py | 97 ++++++++++++++++++++++++++++------------------- policy_mapping.py | 97 ++++++++++++++++++++++++++++------------------- 2 files changed, 114 insertions(+), 80 deletions(-) diff --git a/dev.py b/dev.py index 2cbfb88..3ff2fd2 100644 --- a/dev.py +++ b/dev.py @@ -3,7 +3,7 @@ import httpx import json import logging -from fastapi import FastAPI, Form +from fastapi import FastAPI, Form, Response, status from fastapi.responses import JSONResponse from fuzzywuzzy import fuzz from typing import List, Union @@ -108,6 +108,14 @@ def generate_request_id(): id = uuid.uuid1() return id.hex +#-------------for badrequest--------- +def create_bad_request_response(response_val): + return Response( + content=json.dumps(response_val), + status_code=status.HTTP_400_BAD_REQUEST, + headers={'Content-Type': 'application/json'} + ) + #--------------for adding custome attributes in cymmetri field list------------ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): @@ -311,7 +319,10 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): ]) if data: for document in data: - matching_element_l2 = document.get('key', None) + key = document.get('key', None) + if key is not None and key in l2: + matching_element_l2 = key + #matching_element_l2 = document.get('key', None) synonyms = document.get('synonyms', []) max_similarity = None if synonyms: @@ -507,20 +518,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "APPID_MISSING_ERROR", - response_val["message"] = "Missing 'appId' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "APPID_MISSING_ERROR", + "message": "Missing 'appId' in request" + } + return create_bad_request_response(response_val) elif 'payload' not in data: - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "PAYLOAD_MISSING_ERROR", - response_val["message"] = "Missing 'payload' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "PAYLOAD_MISSING_ERROR", + "message": "Missing 'payload' in request" + } + return create_bad_request_response(response_val) # Validate the format of 'payload' if not isinstance(data['payload'], dict): @@ -532,13 +545,13 @@ async def get_mapped(data: dict, tenant: str = Header(...)): converted_payload[key] = value data['payload'] = converted_payload else: - #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "MUST_BE_DICT_OR_LIST", - response_val["message"] = "payload' must be a dictionary or list" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "MUST_BE_DICT_OR_LIST", + "message": "payload' must be a dictionary or list" + } + return create_bad_request_response(response_val) json_data = data.get('payload') @@ -725,6 +738,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): + logging.debug(f"API call for auto fill policy for create/update user.") + try: body = payload.get("body") policy_mapping = payload.get("policyMapping") @@ -736,8 +751,10 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "errorCode": "BODY_MISSING_ERROR", "message": "Missing 'body' in request" } - raise HTTPException(400, detail=response_val) - + return create_bad_request_response(response_val) + #return Response(content=json.dumps(response_val),status_code=status.HTTP_400_BAD_REQUEST,headers={'Content-Type':'application/json'}) + + elif not policy_mapping: response_val = { "data": None, @@ -745,7 +762,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "errorCode": "POLICY_MAPPING_MISSING_ERROR", "message": "Missing 'policy_mapping' in request" } - raise HTTPException(400, detail=response_val) + return create_bad_request_response(response_val) + #raise HTTPException(400, detail=response_val) mapped_data = {} @@ -767,8 +785,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") @@ -782,22 +799,22 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Check if 'request_id' and 'policyMapList' are present in the request if not policyMapList : - #raise HTTPException(status_code=400, detail="Missing 'payload' in request") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "POLICYMAPLIST_MISSING", - response_val["message"] = "Missing 'policyMapList' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "POLICYMAPLIST_MISSING", + "message": "Missing 'policyMapList' in request" + } + return create_bad_request_response(response_val) elif not request_id : - #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "REQUEST_ID_MISSING", - response_val["message"] = "Missing 'request_id' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "REQUEST_ID_MISSING", + "message": "Missing 'request_id' in request" + } + return create_bad_request_response(response_val) logging.debug(f" The payload is {payload}") diff --git a/policy_mapping.py b/policy_mapping.py index 2cbfb88..3ff2fd2 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -3,7 +3,7 @@ import httpx import json import logging -from fastapi import FastAPI, Form +from fastapi import FastAPI, Form, Response, status from fastapi.responses import JSONResponse from fuzzywuzzy import fuzz from typing import List, Union @@ -108,6 +108,14 @@ def generate_request_id(): id = uuid.uuid1() return id.hex +#-------------for badrequest--------- +def create_bad_request_response(response_val): + return Response( + content=json.dumps(response_val), + status_code=status.HTTP_400_BAD_REQUEST, + headers={'Content-Type': 'application/json'} + ) + #--------------for adding custome attributes in cymmetri field list------------ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): @@ -311,7 +319,10 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): ]) if data: for document in data: - matching_element_l2 = document.get('key', None) + key = document.get('key', None) + if key is not None and key in l2: + matching_element_l2 = key + #matching_element_l2 = document.get('key', None) synonyms = document.get('synonyms', []) max_similarity = None if synonyms: @@ -507,20 +518,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Check if 'appId' and 'payload' are present in the request if 'appId' not in data: - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "APPID_MISSING_ERROR", - response_val["message"] = "Missing 'appId' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "APPID_MISSING_ERROR", + "message": "Missing 'appId' in request" + } + return create_bad_request_response(response_val) elif 'payload' not in data: - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "PAYLOAD_MISSING_ERROR", - response_val["message"] = "Missing 'payload' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "PAYLOAD_MISSING_ERROR", + "message": "Missing 'payload' in request" + } + return create_bad_request_response(response_val) # Validate the format of 'payload' if not isinstance(data['payload'], dict): @@ -532,13 +545,13 @@ async def get_mapped(data: dict, tenant: str = Header(...)): converted_payload[key] = value data['payload'] = converted_payload else: - #raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "MUST_BE_DICT_OR_LIST", - response_val["message"] = "payload' must be a dictionary or list" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "MUST_BE_DICT_OR_LIST", + "message": "payload' must be a dictionary or list" + } + return create_bad_request_response(response_val) json_data = data.get('payload') @@ -725,6 +738,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): + logging.debug(f"API call for auto fill policy for create/update user.") + try: body = payload.get("body") policy_mapping = payload.get("policyMapping") @@ -736,8 +751,10 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "errorCode": "BODY_MISSING_ERROR", "message": "Missing 'body' in request" } - raise HTTPException(400, detail=response_val) - + return create_bad_request_response(response_val) + #return Response(content=json.dumps(response_val),status_code=status.HTTP_400_BAD_REQUEST,headers={'Content-Type':'application/json'}) + + elif not policy_mapping: response_val = { "data": None, @@ -745,7 +762,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "errorCode": "POLICY_MAPPING_MISSING_ERROR", "message": "Missing 'policy_mapping' in request" } - raise HTTPException(400, detail=response_val) + return create_bad_request_response(response_val) + #raise HTTPException(400, detail=response_val) mapped_data = {} @@ -767,8 +785,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") @@ -782,22 +799,22 @@ async def store_data(payload: dict, tenant: str = Header(None)): # Check if 'request_id' and 'policyMapList' are present in the request if not policyMapList : - #raise HTTPException(status_code=400, detail="Missing 'payload' in request") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "POLICYMAPLIST_MISSING", - response_val["message"] = "Missing 'policyMapList' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "POLICYMAPLIST_MISSING", + "message": "Missing 'policyMapList' in request" + } + return create_bad_request_response(response_val) elif not request_id : - #raise HTTPException(status_code=400, detail="Missing 'request_id' in request") - response_val ={} - response_val["data"] = None - response_val["success"] = "false", - response_val["errorCode"] = "REQUEST_ID_MISSING", - response_val["message"] = "Missing 'request_id' in request" - raise HTTPException(status_code=400, detail=response_val) + response_val = { + "data": None, + "success": False, + "errorCode": "REQUEST_ID_MISSING", + "message": "Missing 'request_id' in request" + } + return create_bad_request_response(response_val) logging.debug(f" The payload is {payload}") From 559de06090abe8376429ff6095b924a0002ac243 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Fri, 5 Apr 2024 19:11:42 +0530 Subject: [PATCH 80/91] [Hotfix] Routes updated so as to not have the same ID All routes had the same ID initially --- routes.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routes.yaml b/routes.yaml index 029a4aa..b2fa855 100644 --- a/routes.yaml +++ b/routes.yaml @@ -1,7 +1,7 @@ --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceGetPolicyMappedPOST project: unotech spec: modify: @@ -52,7 +52,7 @@ type: ingress-route --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceMapFieldsToPolicy project: unotech spec: modify: @@ -103,7 +103,7 @@ type: ingress-route --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceSetFeedback project: unotech spec: modify: @@ -151,4 +151,4 @@ spec: version: "" weight: 100 type: ingress-route ---- \ No newline at end of file +--- From 40570b01628582a5da088b3634f3fc7f05483495 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 8 Apr 2024 17:19:26 +0530 Subject: [PATCH 81/91] updated file for businessPhones for array --- dev.py | 271 +++++++++++++++++----------------------------- policy_mapping.py | 21 ++-- 2 files changed, 112 insertions(+), 180 deletions(-) diff --git a/dev.py b/dev.py index a57e7fd..2f0cb41 100644 --- a/dev.py +++ b/dev.py @@ -53,7 +53,6 @@ def ErrorResponseModel(error, code, message, errorCode): def stored_input(tenant: str): return get_collection(tenant, "amaya_input") - #--------------stored policymap for all users---------------- def stored_response(tenant: str): return get_collection(tenant, "amaya_final_output") @@ -193,12 +192,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - # Construct the label using parent keys if path is nested - "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -209,17 +208,19 @@ def explore_json(obj, path=""): "value": obj }) else: + new_path = path + if not path: + path = "root" for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) if isinstance(item, dict) or isinstance(item, list): explore_json(item, new_path) else: datatype = get_data_type(item) distinct_keys_datatypes.append({ "jsonpath": new_path, - "label": f"Index {index}", + "label": path, "datatype": datatype, - "value": item + "value": obj }) def get_data_type(value): @@ -455,48 +456,6 @@ def generate_final_response(similar_elements: List[Dict[str, Union[str, int, flo return final_response -def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: - matched = False - # Perform case-insensitive exact match - for map_entry in policy_mapping: - external_field = map_entry["external"] - internal_field = map_entry["internal"] - if external_field.lower() == field.lower(): - matched = True - print(f"Exact match found: '{field}' -> '{external_field}'") - return external_field, f"${{{external_field}}}" # Use placeholder syntax - - # Perform fuzzy matching if no direct match is found - best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - if score >= 70: # Adjust the threshold as needed - for map_entry in policy_mapping: - if map_entry["internal"].lower() == best_match: - matched = True - print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax - - if not matched: - print(f"No match found for '{field}'") - return field, None # Return original field if no match is found - - -def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: - mapped_nested_data = {} - for field, value in nested_field.items(): - if isinstance(value, dict): - # Recursively map nested fields - mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_nested_data[field] = placeholder - else: - mapped_nested_data[field] = value - return mapped_nested_data - - - #--------------- for mapping the body in body populating api------------------ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: @@ -548,7 +507,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): try: synonyms_collection = get_master_collection("amayaSynonymsMaster") - input_collection = stored_input(tenant) output_collection = stored_response(tenant) subset_collection = stored_policy_mapped(tenant) @@ -558,7 +516,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): # Store the received response directly into the input collection #input_collection.insert_one(data) - #logging.debug("Input respone saved successfully") # Check if 'appId' and 'payload' are present in the request @@ -602,7 +559,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #print("json data is {}", json_data) - json_data_ = extract_user_data(json_data) if json_data_: @@ -613,42 +569,59 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #print("response_data:", response_data) - if isinstance(l1, str): - l1_list = set(convert_string_to_list(l1)) - print("list1: ",l1_list) - else: - l1_list = set(l1) - print("list1: ",l1_list) - - - l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] - - l2_datatypes = { - 'department': 'STRING', - 'employeeId': 'STRING', - 'designation': 'STRING', - 'appUpdatedDate': 'DATETIME', - 'displayName': 'STRING', - 'mobile': 'STRING', - 'country': 'STRING', - 'city': 'STRING', - 'email': 'STRING', - 'end_date': 'DATE', - 'firstName': 'STRING', - 'login': 'INTEGER', - 'lastName': 'STRING', - 'userType': 'STRING', - 'end_date': 'DATE', - 'login': 'INTEGER', - 'userType': 'STRING', - 'dateOfBirth': 'DATE', - 'endDate': 'DATE', - 'startDate': 'DATE', - 'password': 'password', - 'status': 'STRING', - 'profilePicture': 'profilePicture', - 'appUserId': 'STRING', - 'landline': 'STRING' + l1 = [item['label'] for item in response_data] + + if isinstance(l1, str): + l1_list = set(convert_string_to_list(l1)) + logging.info(f"list1: {l1_list}") + else: + #l1_list = remove_underscores_from_set(l1) + l1_list = set(l1) + #l1_list = set(l1_list) + logging.info(f"list1: {l1_list}") + + + + l2 = ['department', 'employeeId', 'designation', 'appUpdatedDate', 'displayName', 'mobile', 'country', 'city', 'email', 'end_date', 'firstName', 'middleName', 'login', 'lastName', 'userType', 'dateOfBirth', 'endDate', 'startDate', 'password', 'status', 'profilePicture', 'appUserId', 'landline'] + + l2_datatypes = { + 'department': 'STRING', + 'employeeId': 'STRING', + 'designation': 'STRING', + 'appUpdatedDate': 'DATETIME', + 'displayName': 'STRING', + 'mobile': 'STRING', + 'country': 'STRING', + 'city': 'STRING', + 'email': 'STRING', + 'end_date': 'DATE', + 'firstName': 'STRING', + 'middleName': 'STRING', + 'login': 'STRING', + 'lastName': 'STRING', + 'userType': 'STRING', + 'end_date': 'DATE', + 'login': 'STRING', + 'userType': 'STRING', + 'dateOfBirth': 'DATE', + 'endDate': 'DATE', + 'startDate': 'DATE', + 'password': 'password', + 'status': 'STRING', + 'profilePicture': 'profilePicture', + 'appUserId': 'STRING', + 'landline': 'STRING' + } + + l2, l2_datatypes, custom_attributes = add_custom_attributes_to_list(l2, l2_datatypes, tenant) + + #print("custom attributes: ", custom_attributes) + for attribute in custom_attributes: + preprocess_attribute = re.sub(r'[^a-zA-Z0-9]', '', attribute).lower() + + new_synonym = { + "synonym": preprocess_attribute, + "score": 1 } synonyms_collection.update_one( {}, @@ -756,57 +729,40 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return ResponseModel(data=data_response, message="Policy mapping generated successfully") else: - l2_list = l2 - - threshold = 60 - - result = compare_lists_with_fuzzy(l1_list, l2_list, threshold) - - final_response = generate_final_response(result['similar_elements'], response_data, l2_datatypes) - final_response_dict = {"final_response": final_response} - - # Assuming 'appId' is present in the received response - appId = data.get("appId") - final_response_dict['appId'] = appId - - output_collection.update_one( - {"appId": appId}, - {"$set": final_response_dict}, - upsert=True - ) - - subset_response = output_collection.aggregate([ - {"$unwind": "$final_response"}, - {"$match": {"final_response.value": {"$ne": None}, "appId": appId}}, - {"$group": { - "_id": "$final_response.attributeName", - "data": {"$first": "$final_response"} - }}, - {"$project": { - "_id": 0, - "jsonPath": "$data.jsonPath", - "attributeName": "$data.attributeName", - "l1_datatype": "$data.l1_datatype", - "l2_matched": "$data.l2_matched", - "l2_datatype": "$data.l2_datatype", - "value": "$data.value", - "similarity_percentage": "$data.similarity_percentage", - "confidence": "$data.confidence" - }} - ]) - - subset_response_data = list(subset_response) - - # Serialize each document into a JSON serializable format - json_serializable_response = [] - for doc in subset_response_data: - json_serializable_doc = { - "jsonPath": doc["jsonPath"], - "attributeName": doc["attributeName"], - "l1_datatype": doc["l1_datatype"], - "l2_matched": doc["l2_matched"], - "l2_datatype": doc["l2_datatype"], - "value": doc["value"] + logging.info(f" Failed to extract the data from the response") + + except HTTPException: + raise + + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") + +#------- Api for body populating---------- +@app.post("/generativeaisrvc/map_fields_to_policy") +async def map_fields_to_policy(payload: Dict[str, Any]): + logging.debug(f"API call for auto fill policy for create/update user.") + + try: + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body: + response_val = { + "data": None, + "success": False, + "errorCode": "BODY_MISSING_ERROR", + "message": "Missing 'body' in request" + } + return create_bad_request_response(response_val) + #return Response(content=json.dumps(response_val),status_code=status.HTTP_400_BAD_REQUEST,headers={'Content-Type':'application/json'}) + + + elif not policy_mapping: + response_val = { + "data": None, + "success": False, + "errorCode": "POLICY_MAPPING_MISSING_ERROR", + "message": "Missing 'policy_mapping' in request" } return create_bad_request_response(response_val) #raise HTTPException(400, detail=response_val) @@ -1086,34 +1042,9 @@ async def store_data(payload: dict, tenant: str = Header(None)): # except Exception as e: # print("faileddddddd") except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/generativeaisrvc/map_fields_to_policy/") -async def map_fields_to_policy(payload: Dict[str, Any]): - body = payload.get("body") - policy_mapping = payload.get("policyMapping") - - if not body or not policy_mapping: - raise HTTPException(status_code=400, detail="Body and policyMapping are required in the payload") - - mapped_data = {} - - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder - else: - mapped_data[field] = value - - return mapped_data - - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.") + #raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index 4fa263d..2f0cb41 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -192,12 +192,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - # Construct the label using parent keys if path is nested - "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -208,17 +208,19 @@ def explore_json(obj, path=""): "value": obj }) else: + new_path = path + if not path: + path = "root" for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) if isinstance(item, dict) or isinstance(item, list): explore_json(item, new_path) else: datatype = get_data_type(item) distinct_keys_datatypes.append({ "jsonpath": new_path, - "label": f"Index {index}", + "label": path, "datatype": datatype, - "value": item + "value": obj }) def get_data_type(value): @@ -567,7 +569,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #print("response_data:", response_data) - l1 = [item['label'] for item in response_data] if isinstance(l1, str): From a98fbd9f230f96cede79f43ac29f9d8fe9bd48cd Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 8 Apr 2024 20:23:37 +0530 Subject: [PATCH 82/91] updated logic for autofill api for accepting anytype of body --- dev.py | 25 +++++++++++++++---------- policy_mapping.py | 25 +++++++++++++++---------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/dev.py b/dev.py index 2f0cb41..f5e12db 100644 --- a/dev.py +++ b/dev.py @@ -767,19 +767,24 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return create_bad_request_response(response_val) #raise HTTPException(400, detail=response_val) + json_data = extract_user_data(body) + json_data = json.dumps(json_data) + json_data_ = json.loads(json_data) + mapped_data = {} - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder + for item in json_data_: + for field, value in item.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - mapped_data[field] = value + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value return mapped_data #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") diff --git a/policy_mapping.py b/policy_mapping.py index 2f0cb41..f5e12db 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -767,19 +767,24 @@ async def map_fields_to_policy(payload: Dict[str, Any]): return create_bad_request_response(response_val) #raise HTTPException(400, detail=response_val) + json_data = extract_user_data(body) + json_data = json.dumps(json_data) + json_data_ = json.loads(json_data) + mapped_data = {} - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder + for item in json_data_: + for field, value in item.items(): + if isinstance(value, dict): + # If the value is a dictionary (nested object), map its nested fields + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - mapped_data[field] = value + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value return mapped_data #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") From b6e90657602519225c646c568fff6e27f0af62ca Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 10 Apr 2024 16:51:31 +0530 Subject: [PATCH 83/91] updated logic for autofill api --- compare_json.py | 91 ++++++++++++++++++++++++++++++++ dev.py | 132 ++++++++++++++++++++++++++++++++++++---------- mapped_data.py | 14 ++--- policy_mapping.py | 132 ++++++++++++++++++++++++++++++++++++---------- 4 files changed, 304 insertions(+), 65 deletions(-) create mode 100644 compare_json.py diff --git a/compare_json.py b/compare_json.py new file mode 100644 index 0000000..ede1abe --- /dev/null +++ b/compare_json.py @@ -0,0 +1,91 @@ +import json + +def compare_json_structure(json1, json2): + if type(json1) != type(json2): + return False # Different data types (dict vs list) + + if isinstance(json1, dict) and isinstance(json2, dict): + return len(json1) == len(json2) # Compare number of keys + + if isinstance(json1, list) and isinstance(json2, list): + if len(json1) != len(json2): + return False + # Recursively compare elements in the list + for item1, item2 in zip(json1, json2): + if not compare_json_structure(item1, item2): + return False + return True + + return False # Not dictionaries or lists + +# Example usage +# json1 = { +# "input": [ +# { +# "_profiles_id": "", +# "emails": "", +# "firstname": "", +# "name": "", +# "password": "", +# "password2": "", +# "realname": "" +# } +# ] +# } + +# json2 = { +# "_profiles_id": "", +# "emails": "${emails}", +# "firstname": "${firstname}", +# "name": "", +# "password": "${password}", +# "password2": "", +# "realname": "" +# } + +json1 = { + "@odata.context": "https: //graph.microsoft.com/v1.0/$metadata#users/$entity", + "businessPhones": [], + "displayName": "${displayName}", + "givenName": "${givenName}", + "jobTitle": "Product Head", + "mailNickname": "manojgulahe", + "mobilePhone": "${mobilePhone}", + "officeLocation": "${officeLocation}", + "preferredLanguage": None, + "surname": "${surname}", + "userPrincipalName": "${userPrincipalName}", + "passwordProfile": { + "forceChangePasswordNextSignIn": False, + "password": "${password}" + }, + "accountEnabled": True +} + +json2 = { + "@odata.context":"https: //graph.microsoft.com/v1.0/$metadata#users/$entity", + "businessPhones":[], + "displayName":"ManojGulahe", + "givenName":"Manoj", + "jobTitle":"Product Head", + "mailNickname":"manojgulahe", + "mobilePhone":"9324418557", + "officeLocation":"IDAM Project", + "preferredLanguage":None, + "surname":"Gulahe", + "userPrincipalName":"manojgulahe@cymmetri.com", + "passwordProfile":{ + "forceChangePasswordNextSignIn":False, + "password":"xWwvJ]6NMw+bWH-d"}, + "accountEnabled":True} + +# if compare_json_structure(json1, json2): +# return True +# print("JSON objects are same structure.") +# else: +# return False +# print("JSON objects are different structures.") + +result = compare_json_structure(json1, json2) + +print("result: ", result) diff --git a/dev.py b/dev.py index f5e12db..48c1b52 100644 --- a/dev.py +++ b/dev.py @@ -470,13 +470,15 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found - best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - if score >= 70: # Adjust the threshold as needed - for map_entry in policy_mapping: - if map_entry["internal"].lower() == best_match: - matched = True - logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + # best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + # logging.debug(f"best match: {best_match}") + # logging.debug(f"score: {score}") + # if score >= 70: # Adjust the threshold as needed + # for map_entry in policy_mapping: + # if map_entry["internal"].lower() == best_match: + # matched = True + # logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + # return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: logging.debug(f"No match found for '{field}'") @@ -500,6 +502,53 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data +#--------for formating the output in specified way +def combine_data(body, mapped_data): + combined_data = {} + + # Iterate over keys in the body + for key, value in body.items(): + if isinstance(value, list) and value: # If value is a non-empty list + # Handle list of dictionaries + combined_data[key] = [] + for item in value: + mapped_item = {} + # Iterate over key-value pairs in each dictionary + for sub_key, sub_value in item.items(): + if sub_value: # Use the original value if it exists + mapped_item[sub_key] = sub_value + elif sub_key in mapped_data: # Use the mapped value if it exists + mapped_item[sub_key] = mapped_data[sub_key] + else: + mapped_item[sub_key] = "" + combined_data[key].append(mapped_item) + else: + # Handle non-list values + combined_data[key] = value if value else mapped_data.get(key, "") + + return combined_data + +#-------for comapring the body and mapped_data for returning the output +def compare_json_structure(json1, json2): + if type(json1) != type(json2): + return False # Different data types (dict vs list) + + if isinstance(json1, dict) and isinstance(json2, dict): + return len(json1) == len(json2) # Compare number of keys + + if isinstance(json1, list) and isinstance(json2, list): + if len(json1) != len(json2): + return False + # Recursively compare elements in the list + for item1, item2 in zip(json1, json2): + if not compare_json_structure(item1, item2): + return False + return True + + return False + + + #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): @@ -517,9 +566,20 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #input_collection.insert_one(data) #logging.debug("Input respone saved successfully") + + + # response_val = { + # "data": None, + # "success": False, + # "errorCode": "", + # "message": "" + # } + + appId = data.get("appId") + payload = data.get("payload") # Check if 'appId' and 'payload' are present in the request - if 'appId' not in data: + if not appId: response_val = { "data": None, "success": False, @@ -528,7 +588,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): } return create_bad_request_response(response_val) - elif 'payload' not in data: + + elif not payload: response_val = { "data": None, "success": False, @@ -538,22 +599,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return create_bad_request_response(response_val) # Validate the format of 'payload' - if not isinstance(data['payload'], dict): - if isinstance(data['payload'], list): - # Convert list of dictionaries to a single dictionary - converted_payload = {} - for item in data['payload']: - for key, value in item.items(): - converted_payload[key] = value - data['payload'] = converted_payload - else: - response_val = { - "data": None, - "success": False, - "errorCode": "MUST_BE_DICT_OR_LIST", - "message": "payload' must be a dictionary or list" - } - return create_bad_request_response(response_val) + # if not isinstance(data['payload'], dict): + # if isinstance(data['payload'], list): + # # Convert list of dictionaries to a single dictionary + # converted_payload = {} + # for item in data['payload']: + # for key, value in item.items(): + # converted_payload[key] = value + # data['payload'] = converted_payload + # else: + # response_val = { + # "data": None, + # "success": False, + # "errorCode": "MUST_BE_DICT_OR_LIST", + # "message": "payload' must be a dictionary or list" + # } + # return create_bad_request_response(response_val) json_data = data.get('payload') @@ -735,7 +796,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") + #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") @@ -771,6 +833,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): json_data = json.dumps(json_data) json_data_ = json.loads(json_data) + print("json_data: ",json_data_) + mapped_data = {} for item in json_data_: @@ -786,14 +850,23 @@ async def map_fields_to_policy(payload: Dict[str, Any]): else: mapped_data[field] = value - return mapped_data - #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") + #print("mapped_data: ", mapped_data) + + result = compare_json_structure(body, mapped_data) + + + if result == False: + combined_data = combine_data(body, mapped_data) + return combined_data + else: + return mapped_data except HTTPException: raise except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") + #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -1052,4 +1125,5 @@ async def store_data(payload: dict, tenant: str = Header(None)): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) + diff --git a/mapped_data.py b/mapped_data.py index ec82398..8f70661 100644 --- a/mapped_data.py +++ b/mapped_data.py @@ -17,13 +17,13 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found - best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - if score >= 70: # Adjust the threshold as needed - for map_entry in policy_mapping: - if map_entry["internal"].lower() == best_match: - matched = True - print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + # best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + # if score >= 70: # Adjust the threshold as needed + # for map_entry in policy_mapping: + # if map_entry["internal"].lower() == best_match: + # matched = True + # print(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + # return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: print(f"No match found for '{field}'") diff --git a/policy_mapping.py b/policy_mapping.py index f5e12db..48c1b52 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -470,13 +470,15 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str return external_field, f"${{{external_field}}}" # Use placeholder syntax # Perform fuzzy matching if no direct match is found - best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - if score >= 70: # Adjust the threshold as needed - for map_entry in policy_mapping: - if map_entry["internal"].lower() == best_match: - matched = True - logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + # best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + # logging.debug(f"best match: {best_match}") + # logging.debug(f"score: {score}") + # if score >= 70: # Adjust the threshold as needed + # for map_entry in policy_mapping: + # if map_entry["internal"].lower() == best_match: + # matched = True + # logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + # return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: logging.debug(f"No match found for '{field}'") @@ -500,6 +502,53 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data +#--------for formating the output in specified way +def combine_data(body, mapped_data): + combined_data = {} + + # Iterate over keys in the body + for key, value in body.items(): + if isinstance(value, list) and value: # If value is a non-empty list + # Handle list of dictionaries + combined_data[key] = [] + for item in value: + mapped_item = {} + # Iterate over key-value pairs in each dictionary + for sub_key, sub_value in item.items(): + if sub_value: # Use the original value if it exists + mapped_item[sub_key] = sub_value + elif sub_key in mapped_data: # Use the mapped value if it exists + mapped_item[sub_key] = mapped_data[sub_key] + else: + mapped_item[sub_key] = "" + combined_data[key].append(mapped_item) + else: + # Handle non-list values + combined_data[key] = value if value else mapped_data.get(key, "") + + return combined_data + +#-------for comapring the body and mapped_data for returning the output +def compare_json_structure(json1, json2): + if type(json1) != type(json2): + return False # Different data types (dict vs list) + + if isinstance(json1, dict) and isinstance(json2, dict): + return len(json1) == len(json2) # Compare number of keys + + if isinstance(json1, list) and isinstance(json2, list): + if len(json1) != len(json2): + return False + # Recursively compare elements in the list + for item1, item2 in zip(json1, json2): + if not compare_json_structure(item1, item2): + return False + return True + + return False + + + #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): @@ -517,9 +566,20 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #input_collection.insert_one(data) #logging.debug("Input respone saved successfully") + + + # response_val = { + # "data": None, + # "success": False, + # "errorCode": "", + # "message": "" + # } + + appId = data.get("appId") + payload = data.get("payload") # Check if 'appId' and 'payload' are present in the request - if 'appId' not in data: + if not appId: response_val = { "data": None, "success": False, @@ -528,7 +588,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): } return create_bad_request_response(response_val) - elif 'payload' not in data: + + elif not payload: response_val = { "data": None, "success": False, @@ -538,22 +599,22 @@ async def get_mapped(data: dict, tenant: str = Header(...)): return create_bad_request_response(response_val) # Validate the format of 'payload' - if not isinstance(data['payload'], dict): - if isinstance(data['payload'], list): - # Convert list of dictionaries to a single dictionary - converted_payload = {} - for item in data['payload']: - for key, value in item.items(): - converted_payload[key] = value - data['payload'] = converted_payload - else: - response_val = { - "data": None, - "success": False, - "errorCode": "MUST_BE_DICT_OR_LIST", - "message": "payload' must be a dictionary or list" - } - return create_bad_request_response(response_val) + # if not isinstance(data['payload'], dict): + # if isinstance(data['payload'], list): + # # Convert list of dictionaries to a single dictionary + # converted_payload = {} + # for item in data['payload']: + # for key, value in item.items(): + # converted_payload[key] = value + # data['payload'] = converted_payload + # else: + # response_val = { + # "data": None, + # "success": False, + # "errorCode": "MUST_BE_DICT_OR_LIST", + # "message": "payload' must be a dictionary or list" + # } + # return create_bad_request_response(response_val) json_data = data.get('payload') @@ -735,7 +796,8 @@ async def get_mapped(data: dict, tenant: str = Header(...)): raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") + #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") @@ -771,6 +833,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): json_data = json.dumps(json_data) json_data_ = json.loads(json_data) + print("json_data: ",json_data_) + mapped_data = {} for item in json_data_: @@ -786,14 +850,23 @@ async def map_fields_to_policy(payload: Dict[str, Any]): else: mapped_data[field] = value - return mapped_data - #return ResponseModel(data=mapped_data, message="auto policy mapped generated successfully") + #print("mapped_data: ", mapped_data) + + result = compare_json_structure(body, mapped_data) + + + if result == False: + combined_data = combine_data(body, mapped_data) + return combined_data + else: + return mapped_data except HTTPException: raise except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") + #-------------------Api fpr storing the admin final policymap for training purpose----------- @app.post("/generativeaisrvc/feedback") async def store_data(payload: dict, tenant: str = Header(None)): @@ -1052,4 +1125,5 @@ async def store_data(payload: dict, tenant: str = Header(None)): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) + From 92ff0b7033b7025f60e958006912563aeb6c2fdb Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 10 Apr 2024 19:28:28 +0530 Subject: [PATCH 84/91] updated logic for feedback api for sorting on attributeName --- dev.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev.py b/dev.py index 48c1b52..2a4efcc 100644 --- a/dev.py +++ b/dev.py @@ -798,7 +798,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") - + #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): @@ -917,10 +917,11 @@ async def store_data(payload: dict, tenant: str = Header(None)): if doc1 and doc2: for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): - # print("policy1: ",policy1) - # print("policy2: ",policy2) + logging.debug(f"policy1: {policy1}") + logging.debug(f"policy2: {policy2}") - if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("attributeName") == policy2.get("attributeName") and policy1.get("l2_matched") != policy2.get("l2_matched"): logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") #Fetching attributeName from doc1 attribute_name1 = policy1.get("attributeName").lower() From f74d9eebc5872a3309da6b3bbd89c3ab6d8cc744 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 11 Apr 2024 11:19:29 +0530 Subject: [PATCH 85/91] updated logic for feedback api for zipping with same attributeName --- dev.py | 364 +++++++++++++++++++++++---------------------- policy_mapping.py | 365 ++++++++++++++++++++++++---------------------- 2 files changed, 373 insertions(+), 356 deletions(-) diff --git a/dev.py b/dev.py index 2a4efcc..57e41b5 100644 --- a/dev.py +++ b/dev.py @@ -915,204 +915,212 @@ async def store_data(payload: dict, tenant: str = Header(None)): #query global collection synonyms_collection = get_master_collection("amayaSynonymsMaster") - if doc1 and doc2: - for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): - logging.debug(f"policy1: {policy1}") - logging.debug(f"policy2: {policy2}") + # Convert policyMapList to dictionaries keyed by attributeName + policy_map_1 = {item['attributeName']: item for item in doc1['policyMapList']} + policy_map_2 = {item['attributeName']: item for item in doc2['policyMapList']} + + # Iterate over common attributeNames + common_keys = set(policy_map_1.keys()) & set(policy_map_2.keys()) + for key in common_keys: + policy1 = policy_map_1[key] + policy2 = policy_map_2[key] + + logging.debug(f"Matching policy1: {policy1}") + logging.debug(f"Matching policy2: {policy2}") - #if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("attributeName") == policy2.get("attributeName") and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") - #Fetching attributeName from doc1 - attribute_name1 = policy1.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name1}") - - # Fetching l2_matched from doc1 - l2_matched1 = policy1.get("l2_matched") - logging.debug(f"l2_matched suggested by AI {l2_matched1}") - - l2matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched given by admin {l2matched2}") - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched1}.synonym": attribute_name1 - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched1}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name1] } - } + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("attributeName") == policy2.get("attributeName") and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") + #Fetching attributeName from doc1 + attribute_name1 = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name1}") + + # Fetching l2_matched from doc1 + l2_matched1 = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched1}") + + l2matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched given by admin {l2matched2}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched1}.synonym": attribute_name1 + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched1}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name1] } } } - }, - { - "$unwind": "$synonyms" } - ] + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name1) + } + ], + upsert= True + ) - global_docs = synonyms_collection.aggregate(pipeline) + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") + logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - score = global_doc['synonyms']['score'] - new_score = score - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched1}.synonym": str(attribute_name1) - }, - { - "$set": { - f"synonyms.{l2_matched1}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name1) - } - ], - upsert= True - ) + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") - logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") - logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - #----------------------for storing new synonyms against the admin l2matched--------------------- - attribute_name2 = policy2.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name2}") - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched by admin {l2_matched2}") - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") - logging.debug(f"End of calculation logic") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"End of calculation logic") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") + attribute_name = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name}") - elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") - attribute_name = policy1.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name}") - - # Fetching l2_matched from doc1 - l2_matched = policy1.get("l2_matched") - logging.debug(f"l2_matched suggested by AI {l2_matched}") - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched}.synonym": attribute_name - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name] } - } + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } } } - }, - { - "$unwind": "$synonyms" } - ] - - global_docs = synonyms_collection.aggregate(pipeline) + }, + { + "$unwind": "$synonyms" + } + ] - for global_doc in global_docs: + global_docs = synonyms_collection.aggregate(pipeline) - synonyms = global_doc.get("synonyms", {}) - if synonyms: + for global_doc in global_docs: - score = global_doc['synonyms']['score'] + synonyms = global_doc.get("synonyms", {}) + if synonyms: - if score is not None and score == 1: - new_score = 1 # If the current score is already 1, keep it unchanged - else: - new_score = score + 0.2 + score = global_doc['synonyms']['score'] - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) - logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") - logging.debug(f"End of calculation logic.") + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty string.") - - attribute_name2 = policy2.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name2}") - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched by admin {l2_matched2}") - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) - logging.debug(f"Inserted new synonym: {new_synonym}.") - logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - else: - logging.debug("no need to analyze and changed.") + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty string.") + + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}.") + logging.debug(f"End of calculation logic.") + + else: + logging.debug("no need to analyze and changed.") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} @@ -1121,7 +1129,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): # except Exception as e: # print("faileddddddd") except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.", errorCode= "Invalid") #raise HTTPException(status_code=500, detail=str(e)) diff --git a/policy_mapping.py b/policy_mapping.py index 48c1b52..57e41b5 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -798,7 +798,7 @@ async def get_mapped(data: dict, tenant: str = Header(...)): except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") - + #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): @@ -915,203 +915,212 @@ async def store_data(payload: dict, tenant: str = Header(None)): #query global collection synonyms_collection = get_master_collection("amayaSynonymsMaster") - if doc1 and doc2: - for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): - # print("policy1: ",policy1) - # print("policy2: ",policy2) + # Convert policyMapList to dictionaries keyed by attributeName + policy_map_1 = {item['attributeName']: item for item in doc1['policyMapList']} + policy_map_2 = {item['attributeName']: item for item in doc2['policyMapList']} + + # Iterate over common attributeNames + common_keys = set(policy_map_1.keys()) & set(policy_map_2.keys()) + for key in common_keys: + policy1 = policy_map_1[key] + policy2 = policy_map_2[key] + + logging.debug(f"Matching policy1: {policy1}") + logging.debug(f"Matching policy2: {policy2}") - if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") - #Fetching attributeName from doc1 - attribute_name1 = policy1.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name1}") - - # Fetching l2_matched from doc1 - l2_matched1 = policy1.get("l2_matched") - logging.debug(f"l2_matched suggested by AI {l2_matched1}") - - l2matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched given by admin {l2matched2}") - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched1}.synonym": attribute_name1 - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched1}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name1] } - } + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("attributeName") == policy2.get("attributeName") and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") + #Fetching attributeName from doc1 + attribute_name1 = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name1}") + + # Fetching l2_matched from doc1 + l2_matched1 = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched1}") + + l2matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched given by admin {l2matched2}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched1}.synonym": attribute_name1 + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched1}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name1] } } } - }, - { - "$unwind": "$synonyms" } - ] + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name1) + } + ], + upsert= True + ) - global_docs = synonyms_collection.aggregate(pipeline) + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") + logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - score = global_doc['synonyms']['score'] - new_score = score - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched1}.synonym": str(attribute_name1) - }, - { - "$set": { - f"synonyms.{l2_matched1}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name1) - } - ], - upsert= True - ) + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") - logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") - logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - #----------------------for storing new synonyms against the admin l2matched--------------------- - attribute_name2 = policy2.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name2}") - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched by admin {l2_matched2}") - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") - logging.debug(f"End of calculation logic") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"End of calculation logic") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") + attribute_name = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name}") - elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") - attribute_name = policy1.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name}") - - # Fetching l2_matched from doc1 - l2_matched = policy1.get("l2_matched") - logging.debug(f"l2_matched suggested by AI {l2_matched}") - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched}.synonym": attribute_name - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name] } - } + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } } } - }, - { - "$unwind": "$synonyms" } - ] - - global_docs = synonyms_collection.aggregate(pipeline) + }, + { + "$unwind": "$synonyms" + } + ] - for global_doc in global_docs: + global_docs = synonyms_collection.aggregate(pipeline) - synonyms = global_doc.get("synonyms", {}) - if synonyms: + for global_doc in global_docs: - score = global_doc['synonyms']['score'] + synonyms = global_doc.get("synonyms", {}) + if synonyms: - if score is not None and score == 1: - new_score = 1 # If the current score is already 1, keep it unchanged - else: - new_score = score + 0.2 + score = global_doc['synonyms']['score'] - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) - logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") - logging.debug(f"End of calculation logic.") + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty string.") - - attribute_name2 = policy2.get("attributeName").lower() - logging.debug(f"attribute_name of the application {attribute_name2}") - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - logging.debug(f"l2_matched by admin {l2_matched2}") - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) - logging.debug(f"Inserted new synonym: {new_synonym}.") - logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - else: - logging.debug("no need to analyze and changed.") + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty string.") + + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}.") + logging.debug(f"End of calculation logic.") + + else: + logging.debug("no need to analyze and changed.") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} @@ -1120,7 +1129,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): # except Exception as e: # print("faileddddddd") except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.") + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.", errorCode= "Invalid") #raise HTTPException(status_code=500, detail=str(e)) From 8afa332e5edc21c1d624b245cf2455a5b80a2e19 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 11 Apr 2024 12:16:37 +0530 Subject: [PATCH 86/91] routes update with identifier for each endpoint --- routes.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.yaml b/routes.yaml index 029a4aa..b31b6be 100644 --- a/routes.yaml +++ b/routes.yaml @@ -1,7 +1,7 @@ --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceGetPolicyMappedPOST project: unotech spec: modify: @@ -52,7 +52,7 @@ type: ingress-route --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceMapFieldsToPolicy project: unotech spec: modify: @@ -103,7 +103,7 @@ type: ingress-route --- api: /v1/config/projects/{project}/routing/ingress/{id} meta: - id: generativeaiservice + id: generativeaiserviceSetFeedback project: unotech spec: modify: From 18ef3c17bccafc45be08be2d68ba46d23330e0a3 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Thu, 11 Apr 2024 16:12:29 +0530 Subject: [PATCH 87/91] updated logic for felicity appliction for autofill api with some properties in extract_user. --- dev.py | 60 +++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/dev.py b/dev.py index 57e41b5..6ad0322 100644 --- a/dev.py +++ b/dev.py @@ -152,7 +152,7 @@ def is_user_data(obj): # Convert keys in the object to lowercase for case-insensitive comparison lower_case_obj_keys = {key.lower() for key in obj.keys()} # Define user data keys in lowercase - user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname', 'name', 'password'} # Check if object contains at least one of the common user data keys, ignoring case return any(key in lower_case_obj_keys for key in user_keys) @@ -469,16 +469,16 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax - # Perform fuzzy matching if no direct match is found - # best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - # logging.debug(f"best match: {best_match}") - # logging.debug(f"score: {score}") - # if score >= 70: # Adjust the threshold as needed - # for map_entry in policy_mapping: - # if map_entry["internal"].lower() == best_match: - # matched = True - # logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - # return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + #Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + logging.debug(f"best match: {best_match}") + logging.debug(f"score: {score}") + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: logging.debug(f"No match found for '{field}'") @@ -503,6 +503,31 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li #--------for formating the output in specified way +# def combine_data(body, mapped_data): +# combined_data = {} + +# # Iterate over keys in the body +# for key, value in body.items(): +# if isinstance(value, list) and value: # If value is a non-empty list +# # Handle list of dictionaries +# combined_data[key] = [] +# for item in value: +# mapped_item = {} +# # Iterate over key-value pairs in each dictionary +# for sub_key, sub_value in item.items(): +# if sub_value: # Use the original value if it exists +# mapped_item[sub_key] = sub_value +# elif sub_key in mapped_data: # Use the mapped value if it exists +# mapped_item[sub_key] = mapped_data[sub_key] +# else: +# mapped_item[sub_key] = "" +# combined_data[key].append(mapped_item) +# else: +# # Handle non-list values +# combined_data[key] = value if value else mapped_data.get(key, "") + +# return combined_data + def combine_data(body, mapped_data): combined_data = {} @@ -524,10 +549,18 @@ def combine_data(body, mapped_data): combined_data[key].append(mapped_item) else: # Handle non-list values - combined_data[key] = value if value else mapped_data.get(key, "") + mapped_item = {} + for sub_key, sub_value in mapped_data.items(): + if sub_key in body: # Use the original value if it exists + mapped_item[sub_key] = body[sub_key] + else: + mapped_item[sub_key] = sub_value + combined_data[key] = mapped_item return combined_data + + #-------for comapring the body and mapped_data for returning the output def compare_json_structure(json1, json2): if type(json1) != type(json2): @@ -850,7 +883,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): else: mapped_data[field] = value - #print("mapped_data: ", mapped_data) + print("mapped_data: ", mapped_data) result = compare_json_structure(body, mapped_data) @@ -1133,6 +1166,7 @@ async def store_data(payload: dict, tenant: str = Header(None)): #raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) From 46df1740654bcf7cb4317be218c69c54b1531cef Mon Sep 17 00:00:00 2001 From: Shreyas Date: Fri, 12 Apr 2024 12:54:50 +0530 Subject: [PATCH 88/91] logic updated for autofill api for replacement of values achieved by mapped_data --- autofill.py | 210 ++++++++++++++++++++++++++++++++++++++++++++++ dev.py | 120 ++++---------------------- policy_mapping.py | 110 +++++++----------------- 3 files changed, 256 insertions(+), 184 deletions(-) create mode 100644 autofill.py diff --git a/autofill.py b/autofill.py new file mode 100644 index 0000000..3c81011 --- /dev/null +++ b/autofill.py @@ -0,0 +1,210 @@ +import uvicorn +from fastapi import FastAPI, Form, Request, HTTPException, Header, Response, status +import httpx +import json +import logging +from fastapi.responses import JSONResponse +from fuzzywuzzy import fuzz +from typing import List, Union +import uvicorn +from typing import List, Dict, Union +from database.connection import get_collection, get_master_collection +from dateutil.parser import parse +from datetime import datetime +from datetime import date +import datetime +import json +from typing import Dict, Any, List +from fuzzywuzzy import process +import uvicorn +import uuid +from typing import Set +import re + +app = FastAPI() + +logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', +) + + +def ResponseModel(data, message, code=200, errorCode=None): + return { + "success": True, + "data": data, + "code": code, + "message": message, + "errorCode": errorCode + } + +def ErrorResponseModel(error, code, message, errorCode): + return { + "data": None, + "success": False, + "code": code, + "message": message, + "errorCode": errorCode, + "error": error + } + +def extract_user_data(response): + try: + logging.debug("Extracting the users from the nested json") + user_data_list = [] + + def is_user_data(obj): + # Convert keys in the object to lowercase for case-insensitive comparison + lower_case_obj_keys = {key.lower() for key in obj.keys()} + # Define user data keys in lowercase + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname', 'name', 'password'} + # Check if object contains at least one of the common user data keys, ignoring case + return any(key in lower_case_obj_keys for key in user_keys) + + def traverse(obj, original_keys=None): + # Recursively traverse the JSON object + if isinstance(obj, dict): + if is_user_data(obj): + # Convert keys in the user data to lowercase or handle as needed + user_data_list.append({original_keys.get(k.lower(), k): v for k, v in obj.items()}) + else: + for key, value in obj.items(): + # Maintain original keys for nested dictionaries + traverse(value, {**original_keys, **{key.lower(): key}}) + elif isinstance(obj, list): + for item in obj: + traverse(item, original_keys) + + traverse(response, {}) + + return user_data_list + except Exception as e: + logging.error(f"Error extracting user data: {e}") + raise HTTPException(status_code=400, detail="INVALID_JSON_DATA") + +#--------------- for mapping the body in body populating api------------------ +def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str: + matched = False + # Perform case-insensitive exact match + for map_entry in policy_mapping: + external_field = map_entry["external"] + internal_field = map_entry["internal"] + if external_field.lower() == field.lower(): + matched = True + logging.debug(f"Exact match found: '{field}' -> '{external_field}'") + return external_field, f"${{{external_field}}}" # Use placeholder syntax + + #Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + logging.debug(f"best match: {best_match}") + logging.debug(f"score: {score}") + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + + if not matched: + logging.debug(f"No match found for '{field}'") + return field, None + + +#------- works on nested conditions also +def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: List[Dict[str, Any]]) -> Dict[str, Any]: + mapped_nested_data = {} + for field, value in nested_field.items(): + if isinstance(value, dict): + # Recursively map nested fields + mapped_nested_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + # Map non-nested fields + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_nested_data[field] = placeholder + else: + mapped_nested_data[field] = value + return mapped_nested_data + + +#-------------for badrequest--------- +def create_bad_request_response(response_val): + return Response( + content=json.dumps(response_val), + status_code=status.HTTP_400_BAD_REQUEST, + headers={'Content-Type': 'application/json'} + ) + +def replace_values_with_placeholders(body, mapped_data): + if isinstance(body, dict): + for key, value in body.items(): + if key in mapped_data: + body[key] = mapped_data[key] + else: + replace_values_with_placeholders(value, mapped_data) + elif isinstance(body, list): + for i, item in enumerate(body): + if isinstance(item, dict) or isinstance(item, list): + replace_values_with_placeholders(item, mapped_data) + return body + +#------- Api for body populating---------- +# Modify the map_fields_to_policy endpoint if needed +@app.post("/generativeaisrvc/map_fields_to_policy") +async def map_fields_to_policy(payload: Dict[str, Any]): + logging.debug(f"API call for auto fill policy for create/update user.") + + try: + body = payload.get("body") + policy_mapping = payload.get("policyMapping") + + if not body: + response_val = { + "data": None, + "success": False, + "errorCode": "BODY_MISSING_ERROR", + "message": "Missing 'body' in request" + } + return create_bad_request_response(response_val) + + elif not policy_mapping: + response_val = { + "data": None, + "success": False, + "errorCode": "POLICY_MAPPING_MISSING_ERROR", + "message": "Missing 'policy_mapping' in request" + } + return create_bad_request_response(response_val) + + json_data = extract_user_data(body) + json_data = json.dumps(json_data) + json_data_ = json.loads(json_data) + + mapped_data = {} + + for item in json_data_: + for field, value in item.items(): + if isinstance(value, dict): + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) + else: + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + data = replace_values_with_placeholders(body, mapped_data) + + + return data + #return ResponseModel(data=data, message="Autofill executed successfully") + + + except HTTPException: + raise + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") + + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file diff --git a/dev.py b/dev.py index 6ad0322..4621325 100644 --- a/dev.py +++ b/dev.py @@ -501,85 +501,19 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li mapped_nested_data[field] = value return mapped_nested_data - -#--------for formating the output in specified way -# def combine_data(body, mapped_data): -# combined_data = {} - -# # Iterate over keys in the body -# for key, value in body.items(): -# if isinstance(value, list) and value: # If value is a non-empty list -# # Handle list of dictionaries -# combined_data[key] = [] -# for item in value: -# mapped_item = {} -# # Iterate over key-value pairs in each dictionary -# for sub_key, sub_value in item.items(): -# if sub_value: # Use the original value if it exists -# mapped_item[sub_key] = sub_value -# elif sub_key in mapped_data: # Use the mapped value if it exists -# mapped_item[sub_key] = mapped_data[sub_key] -# else: -# mapped_item[sub_key] = "" -# combined_data[key].append(mapped_item) -# else: -# # Handle non-list values -# combined_data[key] = value if value else mapped_data.get(key, "") - -# return combined_data - -def combine_data(body, mapped_data): - combined_data = {} - - # Iterate over keys in the body - for key, value in body.items(): - if isinstance(value, list) and value: # If value is a non-empty list - # Handle list of dictionaries - combined_data[key] = [] - for item in value: - mapped_item = {} - # Iterate over key-value pairs in each dictionary - for sub_key, sub_value in item.items(): - if sub_value: # Use the original value if it exists - mapped_item[sub_key] = sub_value - elif sub_key in mapped_data: # Use the mapped value if it exists - mapped_item[sub_key] = mapped_data[sub_key] - else: - mapped_item[sub_key] = "" - combined_data[key].append(mapped_item) - else: - # Handle non-list values - mapped_item = {} - for sub_key, sub_value in mapped_data.items(): - if sub_key in body: # Use the original value if it exists - mapped_item[sub_key] = body[sub_key] - else: - mapped_item[sub_key] = sub_value - combined_data[key] = mapped_item - - return combined_data - - - -#-------for comapring the body and mapped_data for returning the output -def compare_json_structure(json1, json2): - if type(json1) != type(json2): - return False # Different data types (dict vs list) - - if isinstance(json1, dict) and isinstance(json2, dict): - return len(json1) == len(json2) # Compare number of keys - - if isinstance(json1, list) and isinstance(json2, list): - if len(json1) != len(json2): - return False - # Recursively compare elements in the list - for item1, item2 in zip(json1, json2): - if not compare_json_structure(item1, item2): - return False - return True - - return False - +#----------for replacing the values in body +def replace_values_with_placeholders(body, mapped_data): + if isinstance(body, dict): + for key, value in body.items(): + if key in mapped_data: + body[key] = mapped_data[key] + else: + replace_values_with_placeholders(value, mapped_data) + elif isinstance(body, list): + for i, item in enumerate(body): + if isinstance(item, dict) or isinstance(item, list): + replace_values_with_placeholders(item, mapped_data) + return body #----------------------api for policy mapping----------------------------- @@ -601,13 +535,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #logging.debug("Input respone saved successfully") - # response_val = { - # "data": None, - # "success": False, - # "errorCode": "", - # "message": "" - # } - appId = data.get("appId") payload = data.get("payload") @@ -849,9 +776,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "message": "Missing 'body' in request" } return create_bad_request_response(response_val) - #return Response(content=json.dumps(response_val),status_code=status.HTTP_400_BAD_REQUEST,headers={'Content-Type':'application/json'}) - elif not policy_mapping: response_val = { "data": None, @@ -860,39 +785,29 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "message": "Missing 'policy_mapping' in request" } return create_bad_request_response(response_val) - #raise HTTPException(400, detail=response_val) json_data = extract_user_data(body) json_data = json.dumps(json_data) json_data_ = json.loads(json_data) - print("json_data: ",json_data_) - mapped_data = {} for item in json_data_: for field, value in item.items(): if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - # Map non-nested fields mapped_field, placeholder = map_field_to_policy(field, policy_mapping) if placeholder is not None: mapped_data[field] = placeholder else: mapped_data[field] = value + data = replace_values_with_placeholders(body, mapped_data) - print("mapped_data: ", mapped_data) - result = compare_json_structure(body, mapped_data) - + return data + #return ResponseModel(data=data, message="Autofill executed successfully") - if result == False: - combined_data = combine_data(body, mapped_data) - return combined_data - else: - return mapped_data except HTTPException: raise @@ -1168,5 +1083,4 @@ async def store_data(payload: dict, tenant: str = Header(None)): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) - + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/policy_mapping.py b/policy_mapping.py index 57e41b5..4621325 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -152,7 +152,7 @@ def is_user_data(obj): # Convert keys in the object to lowercase for case-insensitive comparison lower_case_obj_keys = {key.lower() for key in obj.keys()} # Define user data keys in lowercase - user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname', 'name', 'password'} # Check if object contains at least one of the common user data keys, ignoring case return any(key in lower_case_obj_keys for key in user_keys) @@ -469,16 +469,16 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax - # Perform fuzzy matching if no direct match is found - # best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) - # logging.debug(f"best match: {best_match}") - # logging.debug(f"score: {score}") - # if score >= 70: # Adjust the threshold as needed - # for map_entry in policy_mapping: - # if map_entry["internal"].lower() == best_match: - # matched = True - # logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") - # return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax + #Perform fuzzy matching if no direct match is found + best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + logging.debug(f"best match: {best_match}") + logging.debug(f"score: {score}") + if score >= 70: # Adjust the threshold as needed + for map_entry in policy_mapping: + if map_entry["internal"].lower() == best_match: + matched = True + logging.debug(f"Fuzzy match found: '{field}' -> '{map_entry['external']}' (Best match: '{best_match}')") + return map_entry['external'], f"${{{map_entry['external']}}}" # Use placeholder syntax if not matched: logging.debug(f"No match found for '{field}'") @@ -501,52 +501,19 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li mapped_nested_data[field] = value return mapped_nested_data - -#--------for formating the output in specified way -def combine_data(body, mapped_data): - combined_data = {} - - # Iterate over keys in the body - for key, value in body.items(): - if isinstance(value, list) and value: # If value is a non-empty list - # Handle list of dictionaries - combined_data[key] = [] - for item in value: - mapped_item = {} - # Iterate over key-value pairs in each dictionary - for sub_key, sub_value in item.items(): - if sub_value: # Use the original value if it exists - mapped_item[sub_key] = sub_value - elif sub_key in mapped_data: # Use the mapped value if it exists - mapped_item[sub_key] = mapped_data[sub_key] - else: - mapped_item[sub_key] = "" - combined_data[key].append(mapped_item) - else: - # Handle non-list values - combined_data[key] = value if value else mapped_data.get(key, "") - - return combined_data - -#-------for comapring the body and mapped_data for returning the output -def compare_json_structure(json1, json2): - if type(json1) != type(json2): - return False # Different data types (dict vs list) - - if isinstance(json1, dict) and isinstance(json2, dict): - return len(json1) == len(json2) # Compare number of keys - - if isinstance(json1, list) and isinstance(json2, list): - if len(json1) != len(json2): - return False - # Recursively compare elements in the list - for item1, item2 in zip(json1, json2): - if not compare_json_structure(item1, item2): - return False - return True - - return False - +#----------for replacing the values in body +def replace_values_with_placeholders(body, mapped_data): + if isinstance(body, dict): + for key, value in body.items(): + if key in mapped_data: + body[key] = mapped_data[key] + else: + replace_values_with_placeholders(value, mapped_data) + elif isinstance(body, list): + for i, item in enumerate(body): + if isinstance(item, dict) or isinstance(item, list): + replace_values_with_placeholders(item, mapped_data) + return body #----------------------api for policy mapping----------------------------- @@ -568,13 +535,6 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #logging.debug("Input respone saved successfully") - # response_val = { - # "data": None, - # "success": False, - # "errorCode": "", - # "message": "" - # } - appId = data.get("appId") payload = data.get("payload") @@ -816,9 +776,7 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "message": "Missing 'body' in request" } return create_bad_request_response(response_val) - #return Response(content=json.dumps(response_val),status_code=status.HTTP_400_BAD_REQUEST,headers={'Content-Type':'application/json'}) - elif not policy_mapping: response_val = { "data": None, @@ -827,39 +785,29 @@ async def map_fields_to_policy(payload: Dict[str, Any]): "message": "Missing 'policy_mapping' in request" } return create_bad_request_response(response_val) - #raise HTTPException(400, detail=response_val) json_data = extract_user_data(body) json_data = json.dumps(json_data) json_data_ = json.loads(json_data) - print("json_data: ",json_data_) - mapped_data = {} for item in json_data_: for field, value in item.items(): if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - # Map non-nested fields mapped_field, placeholder = map_field_to_policy(field, policy_mapping) if placeholder is not None: mapped_data[field] = placeholder else: mapped_data[field] = value + data = replace_values_with_placeholders(body, mapped_data) - #print("mapped_data: ", mapped_data) - result = compare_json_structure(body, mapped_data) - + return data + #return ResponseModel(data=data, message="Autofill executed successfully") - if result == False: - combined_data = combine_data(body, mapped_data) - return combined_data - else: - return mapped_data except HTTPException: raise @@ -1133,6 +1081,6 @@ async def store_data(payload: dict, tenant: str = Header(None)): #raise HTTPException(status_code=500, detail=str(e)) -if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file From 12559489ec0a619a3f00f0e8b9dac2c2bca46def Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 15 Apr 2024 11:20:18 +0530 Subject: [PATCH 89/91] logic added for string/ list replacement for autofill api --- dev.py | 28 +++++++++++++++++++++++++--- policy_mapping.py | 28 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/dev.py b/dev.py index 4621325..93c8d42 100644 --- a/dev.py +++ b/dev.py @@ -502,19 +502,41 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data #----------for replacing the values in body +# def replace_values_with_placeholders(body, mapped_data): +# if isinstance(body, dict): +# for key, value in body.items(): +# if key in mapped_data: +# body[key] = mapped_data[key] +# else: +# replace_values_with_placeholders(value, mapped_data) +# elif isinstance(body, list): +# for i, item in enumerate(body): +# if isinstance(item, dict) or isinstance(item, list): +# replace_values_with_placeholders(item, mapped_data) +# return body + def replace_values_with_placeholders(body, mapped_data): if isinstance(body, dict): for key, value in body.items(): if key in mapped_data: - body[key] = mapped_data[key] + # Check if the original type in body is a list + if isinstance(value, list): + # Replace but maintain the list structure + body[key] = [mapped_data[key]] + else: + # Direct replacement for other types (mostly strings) + body[key] = mapped_data[key] else: + # Recurse into nested structures replace_values_with_placeholders(value, mapped_data) elif isinstance(body, list): for i, item in enumerate(body): - if isinstance(item, dict) or isinstance(item, list): + # Since we handle non-dict or non-list items at a higher level, + # just recursively call the function for dicts and lists + if isinstance(item, (dict, list)): replace_values_with_placeholders(item, mapped_data) - return body + return body #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') diff --git a/policy_mapping.py b/policy_mapping.py index 4621325..93c8d42 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -502,19 +502,41 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li return mapped_nested_data #----------for replacing the values in body +# def replace_values_with_placeholders(body, mapped_data): +# if isinstance(body, dict): +# for key, value in body.items(): +# if key in mapped_data: +# body[key] = mapped_data[key] +# else: +# replace_values_with_placeholders(value, mapped_data) +# elif isinstance(body, list): +# for i, item in enumerate(body): +# if isinstance(item, dict) or isinstance(item, list): +# replace_values_with_placeholders(item, mapped_data) +# return body + def replace_values_with_placeholders(body, mapped_data): if isinstance(body, dict): for key, value in body.items(): if key in mapped_data: - body[key] = mapped_data[key] + # Check if the original type in body is a list + if isinstance(value, list): + # Replace but maintain the list structure + body[key] = [mapped_data[key]] + else: + # Direct replacement for other types (mostly strings) + body[key] = mapped_data[key] else: + # Recurse into nested structures replace_values_with_placeholders(value, mapped_data) elif isinstance(body, list): for i, item in enumerate(body): - if isinstance(item, dict) or isinstance(item, list): + # Since we handle non-dict or non-list items at a higher level, + # just recursively call the function for dicts and lists + if isinstance(item, (dict, list)): replace_values_with_placeholders(item, mapped_data) - return body + return body #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') From 39cced685d90a82131d2db8f6c18d24665ab7599 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 15 Apr 2024 12:05:14 +0530 Subject: [PATCH 90/91] response type updated with succes and message for map_fields_to_policy --- dev.py | 4 ++-- policy_mapping.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev.py b/dev.py index 93c8d42..85dc8c1 100644 --- a/dev.py +++ b/dev.py @@ -827,8 +827,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): data = replace_values_with_placeholders(body, mapped_data) - return data - #return ResponseModel(data=data, message="Autofill executed successfully") + #return data + return ResponseModel(data=data, message="Autofill executed successfully") except HTTPException: diff --git a/policy_mapping.py b/policy_mapping.py index 93c8d42..85dc8c1 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -827,8 +827,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): data = replace_values_with_placeholders(body, mapped_data) - return data - #return ResponseModel(data=data, message="Autofill executed successfully") + #return data + return ResponseModel(data=data, message="Autofill executed successfully") except HTTPException: From b22800242cff3e22ad9ee591c41f9d0f3aafd2dd Mon Sep 17 00:00:00 2001 From: Shreyas Date: Wed, 17 Apr 2024 11:37:56 +0530 Subject: [PATCH 91/91] logic updated for replacement of values in body for autofill for value type list --- backup_dev.py | 636 +++++++++++++++++++++++++++++----------------- dev.py | 38 ++- policy_mapping.py | 28 +- 3 files changed, 445 insertions(+), 257 deletions(-) diff --git a/backup_dev.py b/backup_dev.py index 0fbc617..6cd9dfc 100644 --- a/backup_dev.py +++ b/backup_dev.py @@ -3,7 +3,7 @@ import httpx import json import logging -from fastapi import FastAPI, Form +from fastapi import FastAPI, Form, Response, status from fastapi.responses import JSONResponse from fuzzywuzzy import fuzz from typing import List, Union @@ -30,16 +30,24 @@ ) -def ResponseModel(data, message, code=200, error_code=None): +def ResponseModel(data, message, code=200, errorCode=None): return { + "success": True, "data": data, "code": code, "message": message, - "error_code": error_code + "errorCode": errorCode } -def ErrorResponseModel(error, code, message): - return {"error": error, "code": code, "message": message} +def ErrorResponseModel(error, code, message, errorCode): + return { + "data": None, + "success": False, + "code": code, + "message": message, + "errorCode": errorCode, + "error": error + } #--------- stored the payload as input---------- def stored_input(tenant: str): @@ -100,6 +108,14 @@ def generate_request_id(): id = uuid.uuid1() return id.hex +#-------------for badrequest--------- +def create_bad_request_response(response_val): + return Response( + content=json.dumps(response_val), + status_code=status.HTTP_400_BAD_REQUEST, + headers={'Content-Type': 'application/json'} + ) + #--------------for adding custome attributes in cymmetri field list------------ def add_custom_attributes_to_list(l2, l2_datatypes, tenant): @@ -136,7 +152,7 @@ def is_user_data(obj): # Convert keys in the object to lowercase for case-insensitive comparison lower_case_obj_keys = {key.lower() for key in obj.keys()} # Define user data keys in lowercase - user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname'} + user_keys = {'displayname', 'givenname', 'email', 'id', 'dateofbirth', 'mobile', 'firstname', 'name', 'password'} # Check if object contains at least one of the common user data keys, ignoring case return any(key in lower_case_obj_keys for key in user_keys) @@ -176,12 +192,12 @@ def explore_json(obj, path=""): else: datatype = get_data_type(value) distinct_keys_datatypes.append({ - "jsonpath": new_path, - # Construct the label using parent keys if path is nested - "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, - "datatype": datatype, - "value": value - }) + "jsonpath": new_path, + # Construct the label using parent keys if path is nested + "label": ".".join(new_path.split(".")[1:]) if "." in new_path else key, + "datatype": datatype, + "value": value + }) elif isinstance(obj, list): if not obj: # Check if the list is empty datatype = 'ARRAY' @@ -192,17 +208,19 @@ def explore_json(obj, path=""): "value": obj }) else: + new_path = path + if not path: + path = "root" for index, item in enumerate(obj): - new_path = f"{path}.{index}" if path else str(index) if isinstance(item, dict) or isinstance(item, list): explore_json(item, new_path) else: datatype = get_data_type(item) distinct_keys_datatypes.append({ "jsonpath": new_path, - "label": f"Index {index}", + "label": path, "datatype": datatype, - "value": item + "value": obj }) def get_data_type(value): @@ -303,7 +321,10 @@ def compare_lists_with_fuzzy(l1, l2, threshold, synonyms_collection): ]) if data: for document in data: - matching_element_l2 = document.get('key', None) + key = document.get('key', None) + if key is not None and key in l2: + matching_element_l2 = key + #matching_element_l2 = document.get('key', None) synonyms = document.get('synonyms', []) max_similarity = None if synonyms: @@ -375,6 +396,7 @@ def get_confidence_level(similarity_score: float, score_collection) -> str: return "Unknown" # No matching range found, return Unknown except Exception as e: raise HTTPException(status_code=400, detail="score_collection_error") + #----------------------generates final response--------------- @@ -447,8 +469,10 @@ def map_field_to_policy(field: str, policy_mapping: List[Dict[str, Any]]) -> str logging.debug(f"Exact match found: '{field}' -> '{external_field}'") return external_field, f"${{{external_field}}}" # Use placeholder syntax - # Perform fuzzy matching if no direct match is found + #Perform fuzzy matching if no direct match is found best_match, score = process.extractOne(field.lower(), [map_entry["internal"].lower() for map_entry in policy_mapping]) + logging.debug(f"best match: {best_match}") + logging.debug(f"score: {score}") if score >= 70: # Adjust the threshold as needed for map_entry in policy_mapping: if map_entry["internal"].lower() == best_match: @@ -477,6 +501,70 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li mapped_nested_data[field] = value return mapped_nested_data +#----------for replacing the values in body str replacement +# def replace_values_with_placeholders(body, mapped_data): +# if isinstance(body, dict): +# for key, value in body.items(): +# if key in mapped_data: +# body[key] = mapped_data[key] +# else: +# replace_values_with_placeholders(value, mapped_data) +# elif isinstance(body, list): +# for i, item in enumerate(body): +# if isinstance(item, dict) or isinstance(item, list): +# replace_values_with_placeholders(item, mapped_data) +# return body + +#------------- old function issues with businessPhones logic +# def replace_values_with_placeholders(body, mapped_data): +# if isinstance(body, dict): +# for key, value in body.items(): +# if key in mapped_data: +# # Check if the original type in body is a list +# if isinstance(value, list): +# # Replace but maintain the list structure +# body[key] = mapped_data[key] +# else: +# # Direct replacement for other types (mostly strings) +# body[key] = mapped_data[key] +# else: +# # Recurse into nested structures +# replace_values_with_placeholders(value, mapped_data) +# elif isinstance(body, list): +# for i, item in enumerate(body): +# # Since we handle non-dict or non-list items at a higher level, +# # just recursively call the function for dicts and lists +# if isinstance(item, (dict, list)): +# replace_values_with_placeholders(item, mapped_data) +# +# return body + +#------------for replacing values in body +def replace_values_with_placeholders(body, mapped_data): + if isinstance(body, dict): + for key, value in body.items(): + if key in mapped_data: + # Check if the original type in body is a list + if isinstance(value, list): + # Replace but check if mapped_data[key] is also a list + if isinstance(mapped_data[key], list): + body[key] = mapped_data[key] + else: + # Wrap non-list data from mapped_data in a list + body[key] = [mapped_data[key]] + else: + # Direct replacement for other types + body[key] = mapped_data[key] + else: + # Recurse into nested structures + replace_values_with_placeholders(value, mapped_data) + elif isinstance(body, list): + for i, item in enumerate(body): + if isinstance(item, (dict, list)): + replace_values_with_placeholders(item, mapped_data) + + return body + #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') @@ -495,24 +583,48 @@ async def get_mapped(data: dict, tenant: str = Header(...)): #input_collection.insert_one(data) #logging.debug("Input respone saved successfully") + - # Check if 'appId' and 'payload' are present in the request - if 'appId' not in data: - raise HTTPException(status_code=400, detail="Missing 'appId' in request") - elif 'payload' not in data: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") + appId = data.get("appId") + payload = data.get("payload") + # Check if 'appId' and 'payload' are present in the request + if not appId: + response_val = { + "data": None, + "success": False, + "errorCode": "APPID_MISSING_ERROR", + "message": "Missing 'appId' in request" + } + return create_bad_request_response(response_val) + + + elif not payload: + response_val = { + "data": None, + "success": False, + "errorCode": "PAYLOAD_MISSING_ERROR", + "message": "Missing 'payload' in request" + } + return create_bad_request_response(response_val) + # Validate the format of 'payload' - if not isinstance(data['payload'], dict): - if isinstance(data['payload'], list): - # Convert list of dictionaries to a single dictionary - converted_payload = {} - for item in data['payload']: - for key, value in item.items(): - converted_payload[key] = value - data['payload'] = converted_payload - else: - raise HTTPException(status_code=400, detail="'payload' must be a dictionary or list") + # if not isinstance(data['payload'], dict): + # if isinstance(data['payload'], list): + # # Convert list of dictionaries to a single dictionary + # converted_payload = {} + # for item in data['payload']: + # for key, value in item.items(): + # converted_payload[key] = value + # data['payload'] = converted_payload + # else: + # response_val = { + # "data": None, + # "success": False, + # "errorCode": "MUST_BE_DICT_OR_LIST", + # "message": "payload' must be a dictionary or list" + # } + # return create_bad_request_response(response_val) json_data = data.get('payload') @@ -694,41 +806,65 @@ async def get_mapped(data: dict, tenant: str = Header(...)): raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running policy mappping.", errorCode= "Invalid") + + #------- Api for body populating---------- @app.post("/generativeaisrvc/map_fields_to_policy") async def map_fields_to_policy(payload: Dict[str, Any]): + logging.debug(f"API call for auto fill policy for create/update user.") + try: body = payload.get("body") policy_mapping = payload.get("policyMapping") if not body: - raise HTTPException(status_code=400, detail="body empty") + response_val = { + "data": None, + "success": False, + "errorCode": "BODY_MISSING_ERROR", + "message": "Missing 'body' in request" + } + return create_bad_request_response(response_val) + elif not policy_mapping: - raise HTTPException(status_code=400, detail="policy_mapping empty") + response_val = { + "data": None, + "success": False, + "errorCode": "POLICY_MAPPING_MISSING_ERROR", + "message": "Missing 'policy_mapping' in request" + } + return create_bad_request_response(response_val) + + json_data = extract_user_data(body) + json_data = json.dumps(json_data) + json_data_ = json.loads(json_data) mapped_data = {} - for field, value in body.items(): - if isinstance(value, dict): - # If the value is a dictionary (nested object), map its nested fields - mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) - else: - # Map non-nested fields - mapped_field, placeholder = map_field_to_policy(field, policy_mapping) - if placeholder is not None: - mapped_data[field] = placeholder + for item in json_data_: + for field, value in item.items(): + if isinstance(value, dict): + mapped_data[field] = map_nested_fields_to_policy(value, policy_mapping) else: - mapped_data[field] = value + mapped_field, placeholder = map_field_to_policy(field, policy_mapping) + if placeholder is not None: + mapped_data[field] = placeholder + else: + mapped_data[field] = value + + print("mapped_data: ",mapped_data) + data = replace_values_with_placeholders(body, mapped_data) + + + #return data + return ResponseModel(data=data, message="Autofill executed successfully") + - return mapped_data - except HTTPException: raise except Exception as e: - return ErrorResponseModel(error=str(e), code=500, message="Exception while running mapping field.") - + return ErrorResponseModel(error=str(e), code=500, message="Exception while running autofill policy.", errorCode= "Invalid") #-------------------Api fpr storing the admin final policymap for training purpose----------- @@ -738,11 +874,28 @@ async def store_data(payload: dict, tenant: str = Header(None)): logging.debug(f"working on tenant: {tenant}") try: - # Check if 'request_id' and 'payload' are present in the request - if payload is None: - raise HTTPException(status_code=400, detail="Missing 'payload' in request") - elif 'request_id' not in payload: - raise HTTPException(status_code=400, detail="Missing 'request_id' in request") + request_id = payload.get("request_id") + policyMapList = payload.get("policyMapList") + + # Check if 'request_id' and 'policyMapList' are present in the request + if not policyMapList : + response_val = { + "data": None, + "success": False, + "errorCode": "POLICYMAPLIST_MISSING", + "message": "Missing 'policyMapList' in request" + } + return create_bad_request_response(response_val) + + elif not request_id : + response_val = { + "data": None, + "success": False, + "errorCode": "REQUEST_ID_MISSING", + "message": "Missing 'request_id' in request" + } + return create_bad_request_response(response_val) + logging.debug(f" The payload is {payload}") request_id = payload.get("request_id") @@ -762,209 +915,224 @@ async def store_data(payload: dict, tenant: str = Header(None)): #query global collection synonyms_collection = get_master_collection("amayaSynonymsMaster") - if doc1 and doc2: - for policy1, policy2 in zip(doc1["policyMapList"], doc2["policyMapList"]): - # print("policy1: ",policy1) - # print("policy2: ",policy2) + # Convert policyMapList to dictionaries keyed by attributeName + policy_map_1 = {item['attributeName']: item for item in doc1['policyMapList']} + policy_map_2 = {item['attributeName']: item for item in doc2['policyMapList']} + + # Iterate over common attributeNames + common_keys = set(policy_map_1.keys()) & set(policy_map_2.keys()) + for key in common_keys: + policy1 = policy_map_1[key] + policy2 = policy_map_2[key] + + logging.debug(f"Matching policy1: {policy1}") + logging.debug(f"Matching policy2: {policy2}") - if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") - #Fetching attributeName from doc1 - attribute_name1 = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name1) - - # Fetching l2_matched from doc1 - l2_matched1 = policy1.get("l2_matched") - print("l2_matched suggested by AI: ",l2_matched1) - - l2matched2 = policy2.get("l2_matched") - print("l2_matched given by admin",l2matched2) - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched1}.synonym": attribute_name1 - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched1}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name1] } - } + if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") != policy2.get("l2_matched"): + #if policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("attributeName") == policy2.get("attributeName") and policy1.get("l2_matched") != policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are not equal.") + #Fetching attributeName from doc1 + attribute_name1 = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name1}") + + # Fetching l2_matched from doc1 + l2_matched1 = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched1}") + + l2matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched given by admin {l2matched2}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched1}.synonym": attribute_name1 + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched1}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name1] } } } - }, - { - "$unwind": "$synonyms" } - ] + }, + { + "$unwind": "$synonyms" + } + ] + + global_docs = synonyms_collection.aggregate(pipeline) + + for global_doc in global_docs: + synonyms = global_doc.get("synonyms", {}) + if synonyms: + # Accessing the score and updating it + score = global_doc['synonyms']['score'] + new_score = score - 0.2 + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched1}.synonym": str(attribute_name1) + }, + { + "$set": { + f"synonyms.{l2_matched1}.$[elem].score": float(new_score) + } + }, + array_filters=[ + { + "elem.synonym": str(attribute_name1) + } + ], + upsert= True + ) - global_docs = synonyms_collection.aggregate(pipeline) + logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") + logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - for global_doc in global_docs: - synonyms = global_doc.get("synonyms", {}) - if synonyms: - # Accessing the score and updating it - score = global_doc['synonyms']['score'] - new_score = score - 0.2 - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched1}.synonym": str(attribute_name1) - }, - { - "$set": { - f"synonyms.{l2_matched1}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name1) - } - ], - upsert= True - ) + #----------------------for storing new synonyms against the admin l2matched--------------------- + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") - logging.debug(f"Updated score for {attribute_name1} to {new_score} score, since the suggestion given {l2_matched1} was wrong by AI.") - logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - #----------------------for storing new synonyms against the admin l2matched--------------------- - attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) - logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") - logging.debug(f"End of calculation logic") + logging.debug(f"Inserted new synonym as suggested by admin: {new_synonym}") + logging.debug(f"End of calculation logic") + + elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): + logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") + attribute_name = policy1.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name}") - elif policy1.get("matching_decision") == "synonyms" and policy2.get("matching_decision") == "synonyms" and policy1.get("l2_matched") == policy2.get("l2_matched"): - logging.debug(f" checking and updating score where policy1(AI) and policy2(admin) are equal.") - attribute_name = policy1.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name) - - # Fetching l2_matched from doc1 - l2_matched = policy1.get("l2_matched") - print("l2_matched suggested by AI: ", l2_matched) - - # Finding the attribute in the global collection - pipeline = [ - { - "$match": { - f"synonyms.{l2_matched}.synonym": attribute_name - } - }, - { - "$project": { - "_id": 0, - "synonyms": { - "$filter": { - "input": f"$synonyms.{l2_matched}", - "as": "item", - "cond": { "$eq": ["$$item.synonym", attribute_name] } - } + # Fetching l2_matched from doc1 + l2_matched = policy1.get("l2_matched") + logging.debug(f"l2_matched suggested by AI {l2_matched}") + + # Finding the attribute in the global collection + pipeline = [ + { + "$match": { + f"synonyms.{l2_matched}.synonym": attribute_name + } + }, + { + "$project": { + "_id": 0, + "synonyms": { + "$filter": { + "input": f"$synonyms.{l2_matched}", + "as": "item", + "cond": { "$eq": ["$$item.synonym", attribute_name] } } } - }, - { - "$unwind": "$synonyms" } - ] - - global_docs = synonyms_collection.aggregate(pipeline) + }, + { + "$unwind": "$synonyms" + } + ] - for global_doc in global_docs: + global_docs = synonyms_collection.aggregate(pipeline) - synonyms = global_doc.get("synonyms", {}) - if synonyms: + for global_doc in global_docs: - score = global_doc['synonyms']['score'] + synonyms = global_doc.get("synonyms", {}) + if synonyms: - if score is not None and score == 1: - new_score = 1 # If the current score is already 1, keep it unchanged - else: - new_score = score + 0.2 + score = global_doc['synonyms']['score'] - # Updating the global collection with the new score - synonyms_collection.update_one( - { - f"synonyms.{l2_matched}.synonym": str(attribute_name) - }, + if score is not None and score == 1: + new_score = 1 # If the current score is already 1, keep it unchanged + else: + new_score = score + 0.2 + + # Updating the global collection with the new score + synonyms_collection.update_one( + { + f"synonyms.{l2_matched}.synonym": str(attribute_name) + }, + { + "$set": { + f"synonyms.{l2_matched}.$[elem].score": float(new_score) + } + }, + array_filters=[ { - "$set": { - f"synonyms.{l2_matched}.$[elem].score": float(new_score) - } - }, - array_filters=[ - { - "elem.synonym": str(attribute_name) - } - ], - upsert= True - ) + "elem.synonym": str(attribute_name) + } + ], + upsert= True + ) - logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") - logging.debug(f"End of calculation logic.") + logging.debug(f"Updated score for {attribute_name} to {new_score} score, since the suggestion given {l2_matched} was right by AI.") + logging.debug(f"End of calculation logic.") - else: - logging.debug("No 'synonyms' found in the document.") - - elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": - logging.debug(f" checking and updating where matching decision is empty string.") - - attribute_name2 = policy2.get("attributeName").lower() - print("attribute_name of the application: ",attribute_name2) - - # Fetching l2_matched from doc2 - l2_matched2 = policy2.get("l2_matched") - print("l2_matched by admin: ",l2_matched2) - - new_synonym = { - "synonym": attribute_name2, - "score": 1 - } - synonyms_collection.update_one( - {}, - { - "$addToSet": { - f"synonyms.{l2_matched2}": new_synonym - } - }, - upsert=True - ) - logging.debug(f"Inserted new synonym: {new_synonym}.") - logging.debug(f"End of calculation logic.") + else: + logging.debug("No 'synonyms' found in the document.") - else: - logging.debug("no need to analyze and changed.") + elif policy1.get("matching_decision") == "" and policy2.get("matching_decision") == "" and policy2.get("l2_matched")!= "": + logging.debug(f" checking and updating where matching decision is empty string.") + + attribute_name2 = policy2.get("attributeName").lower() + logging.debug(f"attribute_name of the application {attribute_name2}") + + # Fetching l2_matched from doc2 + l2_matched2 = policy2.get("l2_matched") + logging.debug(f"l2_matched by admin {l2_matched2}") + + new_synonym = { + "synonym": attribute_name2, + "score": 1 + } + synonyms_collection.update_one( + {}, + { + "$addToSet": { + f"synonyms.{l2_matched2}": new_synonym + } + }, + upsert=True + ) + logging.debug(f"Inserted new synonym: {new_synonym}.") + logging.debug(f"End of calculation logic.") + + else: + logging.debug("no need to analyze and changed.") #compare fields and make calculation to update the in global collection return {"message": "Data saved successfully"} + except HTTPException: + raise + # except Exception as e: + # print("faileddddddd") except Exception as e: - print("faileddddddd") - raise HTTPException(status_code=500, detail=str(e)) + return ErrorResponseModel(error=str(e), code=500, message="Exception while running feedback.", errorCode= "Invalid") + #raise HTTPException(status_code=500, detail=str(e)) + + if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=5000, debug=True) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/dev.py b/dev.py index 85dc8c1..08c4a44 100644 --- a/dev.py +++ b/dev.py @@ -515,29 +515,55 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li # replace_values_with_placeholders(item, mapped_data) # return body +# def replace_values_with_placeholders(body, mapped_data): +# if isinstance(body, dict): +# for key, value in body.items(): +# if key in mapped_data: +# # Check if the original type in body is a list +# if isinstance(value, list): +# # Replace but maintain the list structure +# body[key] = mapped_data[key] +# else: +# # Direct replacement for other types (mostly strings) +# body[key] = mapped_data[key] +# else: +# # Recurse into nested structures +# replace_values_with_placeholders(value, mapped_data) +# elif isinstance(body, list): +# for i, item in enumerate(body): +# # Since we handle non-dict or non-list items at a higher level, +# # just recursively call the function for dicts and lists +# if isinstance(item, (dict, list)): +# replace_values_with_placeholders(item, mapped_data) + + return body + def replace_values_with_placeholders(body, mapped_data): if isinstance(body, dict): for key, value in body.items(): if key in mapped_data: # Check if the original type in body is a list if isinstance(value, list): - # Replace but maintain the list structure - body[key] = [mapped_data[key]] + # Replace but check if mapped_data[key] is also a list + if isinstance(mapped_data[key], list): + body[key] = mapped_data[key] + else: + # Wrap non-list data from mapped_data in a list + body[key] = [mapped_data[key]] else: - # Direct replacement for other types (mostly strings) + # Direct replacement for other types body[key] = mapped_data[key] else: # Recurse into nested structures replace_values_with_placeholders(value, mapped_data) elif isinstance(body, list): for i, item in enumerate(body): - # Since we handle non-dict or non-list items at a higher level, - # just recursively call the function for dicts and lists if isinstance(item, (dict, list)): replace_values_with_placeholders(item, mapped_data) return body + #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): @@ -824,6 +850,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): mapped_data[field] = placeholder else: mapped_data[field] = value + + print("mapped_data: ",mapped_data) data = replace_values_with_placeholders(body, mapped_data) diff --git a/policy_mapping.py b/policy_mapping.py index 85dc8c1..c1468b3 100644 --- a/policy_mapping.py +++ b/policy_mapping.py @@ -501,19 +501,6 @@ def map_nested_fields_to_policy(nested_field: Dict[str, Any], policy_mapping: Li mapped_nested_data[field] = value return mapped_nested_data -#----------for replacing the values in body -# def replace_values_with_placeholders(body, mapped_data): -# if isinstance(body, dict): -# for key, value in body.items(): -# if key in mapped_data: -# body[key] = mapped_data[key] -# else: -# replace_values_with_placeholders(value, mapped_data) -# elif isinstance(body, list): -# for i, item in enumerate(body): -# if isinstance(item, dict) or isinstance(item, list): -# replace_values_with_placeholders(item, mapped_data) -# return body def replace_values_with_placeholders(body, mapped_data): if isinstance(body, dict): @@ -521,23 +508,26 @@ def replace_values_with_placeholders(body, mapped_data): if key in mapped_data: # Check if the original type in body is a list if isinstance(value, list): - # Replace but maintain the list structure - body[key] = [mapped_data[key]] + # Replace but check if mapped_data[key] is also a list + if isinstance(mapped_data[key], list): + body[key] = mapped_data[key] + else: + # Wrap non-list data from mapped_data in a list + body[key] = [mapped_data[key]] else: - # Direct replacement for other types (mostly strings) + # Direct replacement for other types body[key] = mapped_data[key] else: # Recurse into nested structures replace_values_with_placeholders(value, mapped_data) elif isinstance(body, list): for i, item in enumerate(body): - # Since we handle non-dict or non-list items at a higher level, - # just recursively call the function for dicts and lists if isinstance(item, (dict, list)): replace_values_with_placeholders(item, mapped_data) return body + #----------------------api for policy mapping----------------------------- @app.post('/generativeaisrvc/get_policy_mapped') async def get_mapped(data: dict, tenant: str = Header(...)): @@ -824,6 +814,8 @@ async def map_fields_to_policy(payload: Dict[str, Any]): mapped_data[field] = placeholder else: mapped_data[field] = value + + print("mapped_data: ",mapped_data) data = replace_values_with_placeholders(body, mapped_data)