LLM Configuration Guide¶

# LLM Provider Selection
LLM_PROVIDER=openai

# OpenAI Configuration
OPENAI_API_KEY=sk-proj-xxxxxxxxxxxxxxxxxxxx
LLM_MODEL=gpt-4o-mini  # or gpt-4, gpt-4-turbo, gpt-3.5-turbo

# Generation Parameters
LLM_TEMPERATURE=0.2
LLM_MAX_TOKENS=500

from green_gov_rag.rag.llm_factory import LLMFactory

# Create LLM instance
llm = LLMFactory.create_llm(
    provider="openai",
    model="gpt-4o-mini",
    temperature=0.2,
    max_tokens=500
)

# Use in RAG chain
from green_gov_rag.rag.rag_chain import RAGChain

rag_chain = RAGChain(
    vector_store=vector_store,
    llm_provider="openai",
    llm_model="gpt-4o-mini"
)

result = rag_chain.query("What are NGER thresholds?")

# Via Azure Portal
1. Navigate to Azure Portal
2. Create Resource → Azure OpenAI
3. Select region (e.g., Australia East)
4. Choose pricing tier (Standard S0)

# In Azure OpenAI Studio
1. Go to Deployments
2. Create New Deployment
3. Model: gpt-4o-mini (or gpt-4, gpt-35-turbo)
4. Deployment name: gpt-4o-mini-deployment
5. Deploy

# From Azure Portal → Your OpenAI Resource → Keys and Endpoint
AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
AZURE_OPENAI_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxx

# LLM Provider Selection
LLM_PROVIDER=azure

# Azure OpenAI Configuration
AZURE_OPENAI_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxx
AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
AZURE_OPENAI_DEPLOYMENT=gpt-4o-mini-deployment  # Your deployment name
AZURE_OPENAI_API_VERSION=2024-02-15-preview     # API version

# Model Selection
LLM_MODEL=gpt-4o-mini  # Base model name

# Generation Parameters
LLM_TEMPERATURE=0.2
LLM_MAX_TOKENS=500

from green_gov_rag.rag.llm_factory import LLMFactory

llm = LLMFactory.create_llm(
    provider="azure",
    model="gpt-4o-mini",
    temperature=0.2,
    max_tokens=500
)

# Environment variables automatically loaded
# Uses AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_API_KEY, AZURE_OPENAI_DEPLOYMENT

# Via AWS Console
1. Navigate to AWS Bedrock
2. Model Access → Manage Model Access
3. Enable: Anthropic Claude 3 Sonnet, Claude 3 Haiku, Amazon Titan
4. Request access (usually instant approval)

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "bedrock:InvokeModel",
        "bedrock:InvokeModelWithResponseStream"
      ],
      "Resource": [
        "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0",
        "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-haiku-20240307-v1:0"
      ]
    }
  ]
}

# LLM Provider Selection
LLM_PROVIDER=bedrock

# AWS Credentials
AWS_ACCESS_KEY_ID=AKIAXXXXXXXXXXXXXXXXX
AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
AWS_REGION=us-east-1  # Bedrock available in us-east-1, us-west-2, eu-west-1

# Model Selection (Bedrock model ID)
BEDROCK_MODEL_ID=anthropic.claude-3-sonnet-20240229-v1:0
# Or: anthropic.claude-3-haiku-20240307-v1:0
# Or: amazon.titan-text-premier-v1:0

# Generation Parameters
LLM_TEMPERATURE=0.2
LLM_MAX_TOKENS=500

llm = LLMFactory.create_llm(
    provider="bedrock",
    model="anthropic.claude-3-sonnet-20240229-v1:0",
    temperature=0.2,
    max_tokens=500
)

# LLM Provider Selection
LLM_PROVIDER=anthropic

# Anthropic Configuration
ANTHROPIC_API_KEY=sk-ant-xxxxxxxxxxxxxxxxxxxx

# Model Selection
LLM_MODEL=claude-3-sonnet-20240229
# Or: claude-3-opus-20240229, claude-3-haiku-20240307

# Generation Parameters
LLM_TEMPERATURE=0.2
LLM_MAX_TOKENS=500

llm = LLMFactory.create_llm(
    provider="anthropic",
    model="claude-3-sonnet-20240229",
    temperature=0.2,
    max_tokens=500
)

# Default: retrieve 5 documents
rag_chain.query(query, k=5)  # ~2000 input tokens

# Optimized: retrieve 3 documents
rag_chain.query(query, k=3)  # ~1200 input tokens (40% reduction)

# Production (high quality)
LLM_MODEL=gpt-4o-mini  # $0.15/1M input

# Cost-optimized (good quality)
LLM_MODEL=gpt-3.5-turbo  # $0.50/1M input (but faster)

# Enable query caching (default: 1 hour TTL)
# Saves ~30-40% of LLM calls

# In .env
CACHE_ENABLED=true
CACHE_TTL=3600  # 1 hour

# Default: 500 tokens output (sufficient for most queries)
LLM_MAX_TOKENS=500

# Short answers: 200 tokens
LLM_MAX_TOKENS=200  # 60% cost reduction on output

# Long explanations: 1000 tokens
LLM_MAX_TOKENS=1000  # Increased cost but more detail

# Process 10 documents per batch (reduce API calls)
tagger.tag_all(documents, batch_size=10)

# Add delay between batches to avoid rate limiting
time.sleep(1)  # 1 second delay

prompt = f"""Answer the query based on the following context:

{context}

Query: {query}

Answer:"""

prompt = f"""You are an expert assistant for Australian environmental and planning regulations.

Answer the query based ONLY on the provided context. Follow these guidelines:

1. Cite specific sections, clauses, or regulations when available
2. Use exact wording from regulations for definitions and requirements
3. Highlight jurisdiction-specific rules (federal vs. state vs. local)
4. If the context doesn't contain enough information, say so explicitly
5. Use Australian English spelling and terminology
6. Include relevant thresholds, dates, and numeric values

Context:
{context}

Query: {query}

Answer:"""

if "nger" in query.lower() or "emissions" in query.lower():
    prompt = f"""You are an expert in Australian greenhouse gas emissions reporting under NGER.

Answer the query using the provided regulatory context. Always:
- Specify emission scopes (Scope 1, 2, 3)
- Include thresholds in tonnes CO2-e
- Cite relevant NGER legislation sections
- Mention reporting deadlines when applicable

Context:
{context}

Query: {query}

Answer:"""

if "tree" in query.lower() or "vegetation" in query.lower():
    prompt = f"""You are an expert in Australian planning and vegetation management regulations.

Answer using the provided context. Always:
- Specify LGA-level requirements
- Cite relevant planning schemes
- Mention permit/approval requirements
- Include clearance thresholds (hectares, tree count)

Context:
{context}

Query: {query}

Answer:"""

prompt = f"""You are a regulatory expert. Answer based on provided context.

Example 1:
Query: What are NGER thresholds?
Answer: Under NGER, facilities must report Scope 1 emissions exceeding 25,000 tonnes CO2-e annually [Section 2.1.3]. Corporate groups report if total emissions exceed 50,000 tonnes CO2-e [Section 4.2].

Example 2:
Query: Can I clear native vegetation in Adelaide?
Answer: In the City of Adelaide (LGA 40070), native vegetation clearance requires approval under the SA Planning and Design Code [Section 5.3.2]. Exemptions apply for trees <5m height or non-significant vegetation [Clause 42(a)].

Now answer this query:
Context:
{context}

Query: {query}

Answer:"""

from langchain_openai import ChatOpenAI

llm = ChatOpenAI(
    model="gpt-4o-mini",
    temperature=0.2,
    max_tokens=500,
    top_p=0.9  # Consider top 90% of probability mass
)

llm = ChatOpenAI(
    model="gpt-4o-mini",
    temperature=0.2,
    max_tokens=500,
    frequency_penalty=0.5  # Penalize repeated tokens
)

llm = LLMFactory.create_llm(
    provider="azure",
    model="gpt-4o-mini",
    temperature=0.2,
    max_tokens=500
)

llm = LLMFactory.create_llm(
    provider="azure",
    model="gpt-4o",
    temperature=0.0,  # Deterministic
    max_tokens=300    # Concise
)

llm = LLMFactory.create_llm(
    provider="anthropic",
    model="claude-3-sonnet-20240229",
    temperature=0.3,  # Slightly more creative
    max_tokens=1000   # Longer responses
)

models = [
    {"provider": "azure", "model": "gpt-4o-mini"},
    {"provider": "azure", "model": "gpt-4o"},
    {"provider": "anthropic", "model": "claude-3-sonnet-20240229"}
]

test_queries = [
    "What are NGER Scope 1 thresholds?",
    "Can I clear native vegetation in Adelaide?",
    "What are Scope 3 Category 4 emissions?"
]

for model_config in models:
    llm = LLMFactory.create_llm(**model_config)
    rag_chain = RAGChain(vector_store=vector_store, llm_provider=model_config["provider"])

    for query in test_queries:
        result = rag_chain.query(query)
        print(f"Model: {model_config['model']}")
        print(f"Answer: {result['result']}")
        print(f"Sources: {len(result['source_documents'])}")
        print("---")

# Track token usage
from langchain.callbacks import get_openai_callback

with get_openai_callback() as cb:
    result = rag_chain.query(query)

    print(f"Input tokens: {cb.prompt_tokens}")
    print(f"Output tokens: {cb.completion_tokens}")
    print(f"Total cost: ${cb.total_cost:.4f}")

# Add exponential backoff
from tenacity import retry, wait_exponential, stop_after_attempt

@retry(wait=wait_exponential(multiplier=1, min=2, max=10), stop=stop_after_attempt(3))
def query_with_retry(query: str):
    return rag_chain.query(query)

# Reduce number of retrieved documents
rag_chain.query(query, k=3)  # Instead of k=5

# Or use a model with longer context
LLM_MODEL=gpt-4o  # 128K context vs 16K for gpt-3.5-turbo

# Increase max_tokens
LLM_MAX_TOKENS=1000  # Instead of 500

# Adjust temperature
LLM_TEMPERATURE=0.3  # Instead of 0.2 (less deterministic)