Python: Bug: Unable to Retrieve Memory for Chat Conversation with Semantic Kernel and Azure Cognitive Search

[shravan-ks-donyati]

Description:
I am encountering issues with the TextMemoryPluginACS-recall function in my Semantic Kernel setup, specifically while attempting to retrieve memory for a chat conversation using Azure Cognitive Search as the memory store. The error indicates an invalid expression related to an $select parameter and a failure to connect to the Azure Cognitive Search endpoint.

Error Details:

The error message highlights an invalid expression:

Invalid expression: Could not find a property named 'Id' on type 'search.document'.
Parameter name: $select

Additionally, there are connectivity issues where the connection to the Azure Cognitive Search endpoint fails:

Cannot connect to host <hidden_url>:443 ssl:default [getaddrinfo failed]

Relevant Code:

• The MainPlanner class initializes a Kernel with Azure services, including AzureChatCompletion, AzureTextEmbedding, and AzureCognitiveSearchMemoryStore.
• The populate_memory and search_memory functions attempt to save and retrieve information in the Azure Cognitive Search index.

Expected Behavior:
The TextMemoryPluginACS-recall function should retrieve relevant memory content from the Azure Cognitive Search store without connection errors or expression issues.

Steps to Reproduce:
Initialize MainPlanner with the correct API keys, endpoints, and configurations.
Use populate_memory() to save chat content into the memory store.
Attempt to retrieve stored memory with the TextMemoryPluginACS-recall function.
Environment:

semantic-kernel==1.14.0
Python version: 3.11
openai==1.54.4
azure-search-documents==11.5.2

Additional Notes:

• All endpoints and API keys are correct.
• No explicit close method is available in SemanticTextMemory, and there are no async session managers in use.
• It appears that TextMemoryPluginACS fails to retrieve saved information, possibly due to a misconfiguration or underlying SDK issue related to $select.

MainPlanner.py

import os
import logging
from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion
from semantic_kernel.kernel import Kernel
from semantic_kernel.planners.function_calling_stepwise_planner import FunctionCallingStepwisePlanner, FunctionCallingStepwisePlannerOptions
from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, AzureTextEmbedding
from plugins.native_plugins.azure_search_plugin import AzureAISearchPlugin
from plugins.native_plugins.bing_search_plugin import BingSearchPlugin
from semantic_kernel.connectors.memory.azure_cognitive_search import AzureCognitiveSearchMemoryStore
from semantic_kernel.memory.semantic_text_memory import SemanticTextMemory
from semantic_kernel.core_plugins.text_memory_plugin import TextMemoryPlugin
from PluginFunctions.chat_operations_plugin import MemoryOperations

class MainPlanner:
    COLLECTION_NAME = "dbot-chat-file-storage"
    MAX_CONTEXT_LENGTH = 128000
    chatId = ""
    memory_collection_id = "generic"

    def __init__(self, chatId: str, plugin_options=None):
        self.kernel = Kernel()
        self.plugin_options = plugin_options or {}
        logging.info(f"Plugin options in ContextPlanner: {self.plugin_options}")
        self.chatId = chatId
    
        self.kernel.add_service(
            AzureChatCompletion(
                service_id="default",
                api_key="<hidden_api_key>",
                deployment_name="gpt-4o-2",
                endpoint="<hidden_endpoint>",
                api_version="2024-02-15-preview"
            )
        )

        self.embedding_gen = AzureTextEmbedding(
            service_id="ada",
            api_key="<hidden_api_key>",
            deployment_name="embed-ada-v2",
            endpoint="<hidden_endpoint>",
            api_version="2024-02-15-preview"
        )
        self.kernel.add_service(self.embedding_gen)

        self.acs_memory_store = AzureCognitiveSearchMemoryStore(
            vector_size=1536,
            search_endpoint="<hidden_search_endpoint>",
            admin_key="<hidden_admin_key>"
        )
        self.memory = SemanticTextMemory(storage=self.acs_memory_store, embeddings_generator=self.embedding_gen)
        self.kernel.add_plugin(TextMemoryPlugin(self.memory), "TextMemoryPluginACS")

    async def populate_memory(self):
        chats_obj = MemoryOperations()
        chat_messages_list = chats_obj.fetch_recent_chats(self.chatId)
        
        for chat in list(chat_messages_list):
            await self.memory.save_information(collection=self.memory_collection_id, id="default", text=chat)

        logging.info("Memory populated for the chat with chat id: %s", self.chatId)

    async def search_memory(self, question: str):
        return await self.memory.search(self.memory_collection_id, question)

    def get_main_planner(self):
        plugins_directory = os.path.join(os.getcwd(), 'plugins', 'native_plugins')
        prompt_plugins_directory = os.path.join(os.getcwd(), 'plugins', 'prompt_template_plugins')
        
        chat_gpt_plugin = self.kernel.add_plugin(parent_directory=prompt_plugins_directory, plugin_name="ChatPlugin")

        if self.plugin_options.get("azure_ai_search_plugin_enabled"):
            azure_ai_search_plugin = self.kernel.add_plugin(
                plugin_name="AzureAISearchPlugin", 
                plugin=AzureAISearchPlugin(), 
                parent_directory=plugins_directory
            )
            logging.info("Azure AI Search Plugin enabled and added to kernel.")

        if self.plugin_options.get("bing_search_plugin_enabled"):
            bing_search_plugin = self.kernel.add_plugin(
                plugin_name="BingSearchPlugin", 
                plugin=BingSearchPlugin(), 
                parent_directory=plugins_directory
            )
            logging.info("Bing Search Plugin enabled and added to kernel.")

        options = FunctionCallingStepwisePlannerOptions(
            max_iterations=10,
            max_tokens=20000,
        )

        planner = FunctionCallingStepwisePlanner(service_id="default", options=options)
        return planner
    
    async def get_main_plan(self, user_query, chat_id=None):
        planner_context = f"{user_query}\n chat_id: {chat_id or 'N/A'}\n index: {self.COLLECTION_NAME}"
        planner = self.get_main_planner()
        try:
            context = await planner.invoke(self.kernel, planner_context)
            return context
        except Exception as e:
            logging.error(f"Error in get_context_plan: {e}")
            return None

test2.py

import asyncio
from planners.main_planner import MainPlanner

async def main():
    plugin_options = {
        "bing_search_plugin_enabled": False,
        "azure_ai_search_plugin_enabled": False
    }
    
    obj = MainPlanner("317a29fb-7e0b-4a9e-b9e2-62fc92b7fca8", plugin_options=plugin_options)
    await obj.populate_memory()
    
    res = await obj.get_main_plan("Give me information about donyati from previous conversations")
    print(res)

asyncio.run(main())
`

Final Answer of AI:
I encountered issues while trying to recall information about 'donyati' from previous conversations in the specified collection. Specifically, I was unable to connect to the information storage. Therefore, I cannot retrieve the requested information at this time.

Error:

    raise HttpResponseError(response=response, model=error)
azure.core.exceptions.HttpResponseError: () Invalid expression: Could not find a property named 'Id' on type 'search.document'.
Parameter name: $select
Code:
Message: Invalid expression: Could not find a property named 'Id' on type 'search.document'.
Parameter name: $select
ERROR:asyncio:Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x0000029C35717750>
ERROR:asyncio:Unclosed connector
connections: ['[(<aiohttp.client_proto.ResponseHandler object at 0x0000029C3571C590>, 87595.453)]']   
connector: <aiohttp.connector.TCPConnector object at 0x0000029C35717410>