Java OpenAI API Integration: Building AI Features in Spring Boot Microservices

Table of Contents

1. OpenAI Java SDK Options: Official SDK vs Spring AI
2. Chat Completions: From Hello World to Production Patterns
3. Function Calling: LLM-Driven Microservice Orchestration
4. Embeddings API: Semantic Search in Your Java Backend
5. Streaming with Spring WebFlux and SSE
6. Rate Limiting, Retry & Circuit Breaker for OpenAI Calls
7. Token Cost Management & Prompt Optimization
8. Testing AI Features: Mocking OpenAI in Unit Tests
9. Key Takeaways

<!-- Option A: Official OpenAI Java SDK (direct, full control) -->
<dependency>
    <groupId>com.openai</groupId>
    <artifactId>openai-java</artifactId>
    <version>2.3.0</version>
</dependency>

<!-- Option B: Spring AI (recommended for Spring Boot — portable abstraction) -->
<dependency>
    <groupId>org.springframework.ai</groupId>
    <artifactId>spring-ai-openai-spring-boot-starter</artifactId>
    <version>1.0.0</version>
</dependency>

@Configuration
public class OpenAiConfig {

    @Value("${openai.api-key}")
    private String apiKey;

    @Bean
    public OpenAIClient openAiClient() {
        return OpenAIOkHttpClient.builder()
            .apiKey(apiKey)
            .build();
    }
}

@Service
@RequiredArgsConstructor
public class ChatCompletionService {

    private final OpenAIClient client;

    public String complete(String systemPrompt, String userMessage) {
        ChatCompletion completion = client.chat().completions().create(
            ChatCompletionCreateParams.builder()
                .model(ChatModel.GPT_4O)
                .maxCompletionTokens(1024)
                .addSystemMessage(systemPrompt)
                .addUserMessage(userMessage)
                .build()
        );
        return completion.choices().get(0).message().content().orElse("");
    }

    // Multi-turn conversation with message history
    public String continueConversation(List<ChatCompletionMessageParam> history, String newMessage) {
        List<ChatCompletionMessageParam> messages = new ArrayList<>(history);
        messages.add(ChatCompletionUserMessageParam.builder()
            .content(newMessage).build());

        ChatCompletion completion = client.chat().completions().create(
            ChatCompletionCreateParams.builder()
                .model(ChatModel.GPT_4O)
                .messages(messages)
                .build()
        );
        return completion.choices().get(0).message().content().orElse("");
    }
}

@Service
@RequiredArgsConstructor
public class OrderAssistantService {

    private final OpenAIClient client;
    private final OrderService orderService;
    private final ObjectMapper objectMapper;

    // Tool definition with JSON schema
    private static final ChatCompletionToolParam GET_ORDER_TOOL =
        ChatCompletionToolParam.builder()
            .type(ChatCompletionToolParam.Type.FUNCTION)
            .function(FunctionDefinition.builder()
                .name("getOrder")
                .description("Get details of a customer order by order ID")
                .parameters(FunctionParameters.builder()
                    .putAdditionalProperty("type", JsonValue.from("object"))
                    .putAdditionalProperty("properties", JsonValue.from(Map.of(
                        "orderId", Map.of("type", "string", "description", "The order ID")
                    )))
                    .putAdditionalProperty("required", JsonValue.from(List.of("orderId")))
                    .build())
                .build())
            .build();

    public String handleCustomerQuery(String userQuery) throws Exception {
        List<ChatCompletionMessageParam> messages = new ArrayList<>();
        messages.add(ChatCompletionUserMessageParam.builder().content(userQuery).build());

        // First API call: model decides to use a tool
        ChatCompletion response = client.chat().completions().create(
            ChatCompletionCreateParams.builder()
                .model(ChatModel.GPT_4O)
                .messages(messages)
                .tools(List.of(GET_ORDER_TOOL))
                .build()
        );

        ChatCompletionMessage assistantMessage = response.choices().get(0).message();

        // If model called a tool, execute it and send result back
        if (!assistantMessage.toolCalls().isEmpty()) {
            messages.add(assistantMessage.toParam());
            for (ChatCompletionMessageToolCall toolCall : assistantMessage.toolCalls()) {
                String result = executeToolCall(toolCall);
                messages.add(ChatCompletionToolMessageParam.builder()
                    .toolCallId(toolCall.id())
                    .content(result)
                    .build());
            }
            // Second API call: model generates natural language response with tool result
            ChatCompletion finalResponse = client.chat().completions().create(
                ChatCompletionCreateParams.builder()
                    .model(ChatModel.GPT_4O)
                    .messages(messages)
                    .build()
            );
            return finalResponse.choices().get(0).message().content().orElse("");
        }
        return assistantMessage.content().orElse("");
    }

    private String executeToolCall(ChatCompletionMessageToolCall toolCall) throws Exception {
        if ("getOrder".equals(toolCall.function().name())) {
            Map<String, String> args = objectMapper.readValue(
                toolCall.function().arguments(), new TypeReference<>() {});
            Order order = orderService.findById(args.get("orderId"));
            return objectMapper.writeValueAsString(order);
        }
        return "{\"error\": \"Unknown tool\"}";
    }
}

@Service
@RequiredArgsConstructor
public class SemanticSearchService {

    private final OpenAIClient client;
    private final ProductRepository productRepository; // stores embeddings in PostgreSQL

    // Generate embedding for a single text
    public List<Double> embed(String text) {
        CreateEmbeddingResponse response = client.embeddings().create(
            EmbeddingCreateParams.builder()
                .model(EmbeddingModel.TEXT_EMBEDDING_3_SMALL)
                .input(EmbeddingCreateParams.Input.ofString(text))
                .build()
        );
        return response.data().get(0).embedding();
    }

    // Index products: called once during ingestion
    public void indexProduct(Product product) {
        List<Double> embedding = embed(product.getName() + " " + product.getDescription());
        productRepository.saveWithEmbedding(product, embedding); // pgvector column
    }

    // Semantic search: find products similar to the query
    public List<Product> search(String query, int topK) {
        List<Double> queryEmbedding = embed(query);
        // Uses pgvector cosine similarity: SELECT * FROM products ORDER BY embedding <=> $1 LIMIT $2
        return productRepository.findBySemanticSimilarity(queryEmbedding, topK);
    }
}

@RestController
@RequiredArgsConstructor
public class StreamingChatController {

    private final OpenAIClient client;

    @GetMapping(value = "/chat/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
    public Flux<String> streamChat(@RequestParam String message) {
        return Flux.create(sink -> {
            try (StreamResponse<ChatCompletionChunk> stream =
                    client.chat().completions().createStreaming(
                        ChatCompletionCreateParams.builder()
                            .model(ChatModel.GPT_4O)
                            .addUserMessage(message)
                            .build())) {

                stream.forEach(chunk -> {
                    String content = chunk.choices().get(0).delta().content().orElse("");
                    if (!content.isEmpty()) {
                        sink.next(content);
                    }
                });
                sink.complete();
            } catch (Exception e) {
                sink.error(e);
            }
        });
    }
}

# application.properties
resilience4j.retry.instances.openai.max-attempts=3
resilience4j.retry.instances.openai.wait-duration=2s
resilience4j.retry.instances.openai.exponential-backoff-multiplier=2
resilience4j.retry.instances.openai.retry-exceptions=com.openai.errors.RateLimitError

resilience4j.circuitbreaker.instances.openai.failure-rate-threshold=50
resilience4j.circuitbreaker.instances.openai.wait-duration-in-open-state=30s
resilience4j.circuitbreaker.instances.openai.sliding-window-size=10

@Service
@RequiredArgsConstructor
public class ResilientAiService {

    private final ChatCompletionService delegate;

    @Retry(name = "openai", fallbackMethod = "fallbackResponse")
    @CircuitBreaker(name = "openai", fallbackMethod = "fallbackResponse")
    @RateLimiter(name = "openai")
    public String complete(String systemPrompt, String userMessage) {
        return delegate.complete(systemPrompt, userMessage);
    }

    public String fallbackResponse(String systemPrompt, String userMessage, Exception ex) {
        log.warn("OpenAI unavailable, using fallback. Error: {}", ex.getMessage());
        return "I'm temporarily unable to process your request. Please try again in a moment.";
    }
}

@Service
@RequiredArgsConstructor
public class TokenTrackingService {

    private final MeterRegistry meterRegistry;
    private final OpenAIClient client;
    private final RedisTemplate<String, String> redisTemplate;

    public String completeWithTracking(String prompt, String userId) {
        // Check cache first — identical prompts return cached response
        String cacheKey = "openai:" + DigestUtils.md5Hex(prompt);
        String cached = redisTemplate.opsForValue().get(cacheKey);
        if (cached != null) {
            meterRegistry.counter("ai.cache.hit").increment();
            return cached;
        }

        ChatCompletion response = client.chat().completions().create(
            ChatCompletionCreateParams.builder()
                .model(ChatModel.GPT_4O_MINI) // 15x cheaper than GPT-4o for simple tasks
                .addUserMessage(prompt)
                .maxCompletionTokens(500)     // Hard limit to prevent runaway costs
                .build()
        );

        // Track token usage per user for cost attribution
        long inputTokens = response.usage().map(u -> u.promptTokens()).orElse(0L);
        long outputTokens = response.usage().map(u -> u.completionTokens()).orElse(0L);
        meterRegistry.counter("ai.tokens.input", "user", userId).increment(inputTokens);
        meterRegistry.counter("ai.tokens.output", "user", userId).increment(outputTokens);

        String result = response.choices().get(0).message().content().orElse("");
        redisTemplate.opsForValue().set(cacheKey, result, Duration.ofHours(1));
        return result;
    }
}

@ExtendWith(MockitoExtension.class)
class OrderAssistantServiceTest {

    @Mock
    private OpenAIClient mockClient;

    @Mock
    private OrderService orderService;

    @InjectMocks
    private OrderAssistantService service;

    @Test
    void handleCustomerQuery_withValidOrderId_returnsOrderDetails() {
        // Arrange: mock a response with a tool call
        ChatCompletionMessageToolCall toolCall = mock(ChatCompletionMessageToolCall.class);
        when(toolCall.id()).thenReturn("call_abc123");
        when(toolCall.function().name()).thenReturn("getOrder");
        when(toolCall.function().arguments()).thenReturn("{\"orderId\":\"ORD-001\"}");

        // ... (full mock setup for ChatCompletion response)
        // ... (mock final response after tool call)

        Order mockOrder = new Order("ORD-001", "Laptop", OrderStatus.SHIPPED);
        when(orderService.findById("ORD-001")).thenReturn(mockOrder);

        // Act
        String result = service.handleCustomerQuery("What is the status of my order ORD-001?");

        // Assert
        assertThat(result).isNotBlank();
        verify(orderService).findById("ORD-001");
    }
}

// Integration test: use Spring AI's MockChatModel for full context
@SpringBootTest
class AiControllerIntegrationTest {

    @TestConfiguration
    static class MockAiConfig {
        @Bean
        @Primary
        ChatModel mockChatModel() {
            // Returns deterministic responses for testing
            return new MockChatModel(new ChatResponse(List.of(
                new Generation(new AssistantMessage("Mock response")))));
        }
    }
}

Leave a Comment

Building OpenAI features in Java? Share your experience, architecture questions, or cost optimization tips below.