Skip to content

Commit 8936454

Browse files
peterbandapeterbanda
committed
Fireworks - document inlining
1 parent 8f160cc commit 8936454

7 files changed

+205
-47
lines changed

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/BufferedImageHelper.scala

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package io.cequence.openaiscala.examples
2+
3+
import java.awt.image.RenderedImage
4+
import java.io.ByteArrayOutputStream
5+
import java.util.Base64
6+
import javax.imageio.ImageIO
7+
8+
trait BufferedImageHelper {
9+
10+
protected def imageBase64Source(
11+
file: java.io.File
12+
): String = {
13+
val bufferedImage = ImageIO.read(file)
14+
Base64.getEncoder.encodeToString(imageToBytes(bufferedImage, "jpeg"))
15+
}
16+
17+
protected def imageToBytes(
18+
image: RenderedImage,
19+
format: String
20+
): Array[Byte] = {
21+
val baos = new ByteArrayOutputStream()
22+
ImageIO.write(image, format, baos)
23+
baos.flush()
24+
val imageInByte = baos.toByteArray
25+
baos.close()
26+
imageInByte
27+
}
28+
}

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/CreateChatCompletionVisionWithLocalFile.scala

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,20 @@ package io.cequence.openaiscala.examples
33
import io.cequence.openaiscala.domain._
44
import io.cequence.openaiscala.domain.settings.CreateChatCompletionSettings
55

6-
import java.awt.image.RenderedImage
7-
import java.io.ByteArrayOutputStream
8-
import java.util.Base64
9-
import javax.imageio.ImageIO
106
import scala.concurrent.Future
117

12-
object CreateChatCompletionVisionWithLocalFile extends Example {
8+
object CreateChatCompletionVisionWithLocalFile extends Example with BufferedImageHelper {
139

1410
// provide a local jpeg here
15-
private val localImagePath = sys.env("EXAMPLE_IMAGE_PATH")
16-
private val bufferedImage = ImageIO.read(new java.io.File(localImagePath))
17-
private val imageBase64Source =
18-
Base64.getEncoder.encodeToString(imageToBytes(bufferedImage, "jpeg"))
11+
private lazy val localImagePath = sys.env("EXAMPLE_IMAGE_PATH")
12+
private val imageSource = imageBase64Source(new java.io.File(localImagePath))
1913

2014
val messages: Seq[BaseMessage] = Seq(
2115
SystemMessage("You are a helpful assistant."),
2216
UserSeqMessage(
2317
Seq(
2418
TextContent("What is in this picture?"),
25-
ImageURLContent(s"data:image/jpeg;base64,${imageBase64Source}")
19+
ImageURLContent(s"data:image/jpeg;base64,${imageSource}")
2620
)
2721
)
2822
)
@@ -32,22 +26,10 @@ object CreateChatCompletionVisionWithLocalFile extends Example {
3226
.createChatCompletion(
3327
messages,
3428
settings = CreateChatCompletionSettings(
35-
model = ModelId.gpt_4_vision_preview,
29+
model = ModelId.gpt_4o,
3630
temperature = Some(0),
3731
max_tokens = Some(300)
3832
)
3933
)
4034
.map(printMessageContent)
41-
42-
private def imageToBytes(
43-
image: RenderedImage,
44-
format: String
45-
): Array[Byte] = {
46-
val baos = new ByteArrayOutputStream()
47-
ImageIO.write(image, format, baos)
48-
baos.flush()
49-
val imageInByte = baos.toByteArray
50-
baos.close()
51-
imageInByte
52-
}
5335
}

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/AnthropicCreateMessageWithImage.scala

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,29 +8,25 @@ import io.cequence.openaiscala.anthropic.domain.response.CreateMessageResponse
88
import io.cequence.openaiscala.anthropic.domain.settings.AnthropicCreateMessageSettings
99
import io.cequence.openaiscala.anthropic.service.{AnthropicService, AnthropicServiceFactory}
1010
import io.cequence.openaiscala.domain.NonOpenAIModelId
11-
import io.cequence.openaiscala.examples.ExampleBase
11+
import io.cequence.openaiscala.examples.{BufferedImageHelper, ExampleBase}
1212

13-
import java.awt.image.RenderedImage
14-
import java.io.ByteArrayOutputStream
15-
import java.util.Base64
16-
import javax.imageio.ImageIO
1713
import scala.concurrent.Future
1814

1915
// requires `openai-scala-anthropic-client` as a dependency
20-
object AnthropicCreateMessageWithImage extends ExampleBase[AnthropicService] {
16+
object AnthropicCreateMessageWithImage
17+
extends ExampleBase[AnthropicService]
18+
with BufferedImageHelper {
2119

22-
private val localImagePath = sys.env("EXAMPLE_IMAGE_PATH")
23-
private val bufferedImage = ImageIO.read(new java.io.File(localImagePath))
24-
private val imageBase64Source =
25-
Base64.getEncoder.encodeToString(imageToBytes(bufferedImage, "jpeg"))
20+
private lazy val localImagePath = sys.env("EXAMPLE_IMAGE_PATH")
21+
private val imageSource = imageBase64Source(new java.io.File(localImagePath))
2622

2723
override protected val service: AnthropicService = AnthropicServiceFactory()
2824

2925
private val messages: Seq[Message] = Seq(
3026
UserMessageContent(
3127
Seq(
3228
ContentBlockBase(TextBlock("Describe to me what is in the picture!")),
33-
MediaBlock.jpeg(data = imageBase64Source)
29+
MediaBlock.jpeg(data = imageSource)
3430
)
3531
)
3632
)
@@ -46,18 +42,6 @@ object AnthropicCreateMessageWithImage extends ExampleBase[AnthropicService] {
4642
)
4743
.map(printMessageContent)
4844

49-
private def imageToBytes(
50-
image: RenderedImage,
51-
format: String
52-
): Array[Byte] = {
53-
val baos = new ByteArrayOutputStream()
54-
ImageIO.write(image, format, baos)
55-
baos.flush()
56-
val imageInByte = baos.toByteArray
57-
baos.close()
58-
imageInByte
59-
}
60-
6145
private def printMessageContent(response: CreateMessageResponse) = {
6246
val text =
6347
response.content.blocks.collect { case ContentBlockBase(TextBlock(text), _) => text }

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/CerebrasCreateChatCompletion.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ object CerebrasCreateChatCompletion extends ExampleBase[OpenAIChatCompletionServ
2424
UserMessage("What is the weather like in Norway?")
2525
)
2626

27-
private val modelId = NonOpenAIModelId.llama3_1_8b
27+
private val modelId = NonOpenAIModelId.llama_3_3_70b
2828

2929
override protected def run: Future[_] =
3030
service

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/FireworksAIDocumentInlining.scala

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package io.cequence.openaiscala.examples.nonopenai
2+
3+
import io.cequence.openaiscala.domain._
4+
import io.cequence.openaiscala.domain.settings.CreateChatCompletionSettings
5+
import io.cequence.openaiscala.examples.ExampleBase
6+
import io.cequence.openaiscala.service.OpenAIChatCompletionService
7+
8+
import scala.concurrent.Future
9+
10+
/**
11+
* Requires `FIREWORKS_API_KEY` environment variable to be set
12+
*
13+
* Check out the website for more information:
14+
* https://fireworks.ai/blog/document-inlining-launch
15+
*/
16+
object FireworksAIDocumentInlining extends ExampleBase[OpenAIChatCompletionService] {
17+
18+
private val fireworksModelPrefix = "accounts/fireworks/models/"
19+
override val service: OpenAIChatCompletionService = ChatCompletionProvider.fireworks
20+
21+
val messages: Seq[BaseMessage] = Seq(
22+
SystemMessage("You are a helpful assistant."),
23+
UserSeqMessage(
24+
Seq(
25+
TextContent("What are the candidate's BA and MBA GPAs?"),
26+
ImageURLContent(
27+
"https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf#transform=inline"
28+
)
29+
)
30+
)
31+
)
32+
33+
override protected def run: Future[_] =
34+
service
35+
.createChatCompletion(
36+
messages,
37+
settings = CreateChatCompletionSettings(
38+
model = fireworksModelPrefix + NonOpenAIModelId.llama_v3p3_70b_instruct,
39+
temperature = Some(0),
40+
max_tokens = Some(1000)
41+
)
42+
)
43+
.map(printMessageContent)
44+
}

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/FireworksAIDocumentInliningJson.scala

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package io.cequence.openaiscala.examples.nonopenai
2+
3+
import io.cequence.openaiscala.domain._
4+
import io.cequence.openaiscala.domain.settings.{
5+
ChatCompletionResponseFormatType,
6+
CreateChatCompletionSettings
7+
}
8+
import io.cequence.openaiscala.examples.ExampleBase
9+
import io.cequence.openaiscala.service.OpenAIChatCompletionService
10+
import play.api.libs.json.Json
11+
import io.cequence.openaiscala.JsonFormats.jsonSchemaFormat
12+
13+
import scala.concurrent.Future
14+
15+
/**
16+
* Requires `FIREWORKS_API_KEY` environment variable to be set
17+
*
18+
* Check out the website for more information:
19+
* https://fireworks.ai/blog/document-inlining-launch
20+
*/
21+
object FireworksAIDocumentInliningJson extends ExampleBase[OpenAIChatCompletionService] {
22+
23+
private val fireworksModelPrefix = "accounts/fireworks/models/"
24+
override val service: OpenAIChatCompletionService = ChatCompletionProvider.fireworks
25+
26+
val messages: Seq[BaseMessage] = Seq(
27+
SystemMessage("You are a helpful assistant."),
28+
UserSeqMessage(
29+
Seq(
30+
TextContent(
31+
"Extract the list of professional associations and accomplishments into JSON"
32+
),
33+
ImageURLContent(
34+
"https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf#transform=inline"
35+
)
36+
)
37+
)
38+
)
39+
40+
private val schema: JsonSchema = JsonSchema.Object(
41+
properties = Seq(
42+
"professional_associations" -> JsonSchema.Array(JsonSchema.String()),
43+
"accomplishment" -> JsonSchema.Array(JsonSchema.String())
44+
),
45+
required = Seq("professional_associations", "accomplishment")
46+
)
47+
48+
override protected def run: Future[_] =
49+
service
50+
.createChatCompletion(
51+
messages,
52+
settings = CreateChatCompletionSettings(
53+
model = fireworksModelPrefix + NonOpenAIModelId.llama_v3p3_70b_instruct,
54+
temperature = Some(0),
55+
max_tokens = Some(1000),
56+
// response_format_type = Some(ChatCompletionResponseFormatType.json_object),
57+
extra_params = Map(
58+
"response_format" -> Json.obj(
59+
"type" -> ChatCompletionResponseFormatType.json_object.toString,
60+
"schema" -> Json.toJson(schema)
61+
)
62+
)
63+
)
64+
)
65+
.map(printMessageContent)
66+
}

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/FireworksAIDocumentInliningLocal.scala

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package io.cequence.openaiscala.examples.nonopenai
2+
3+
import io.cequence.openaiscala.domain._
4+
import io.cequence.openaiscala.domain.settings.CreateChatCompletionSettings
5+
import io.cequence.openaiscala.examples.ExampleBase
6+
import io.cequence.openaiscala.service.OpenAIChatCompletionService
7+
8+
import java.nio.file.{Files, Paths}
9+
import java.util.Base64
10+
import scala.concurrent.Future
11+
12+
/**
13+
* Requires `FIREWORKS_API_KEY` and `EXAMPLE_PDF_PATH` environment variables to be set
14+
*
15+
* Check out the website for more information:
16+
* https://fireworks.ai/blog/document-inlining-launch
17+
*/
18+
object FireworksAIDocumentInliningLocal extends ExampleBase[OpenAIChatCompletionService] {
19+
20+
private lazy val localPdfPath = sys.env("EXAMPLE_PDF_PATH")
21+
22+
private val base64Pdf = {
23+
val pdfBytes = Files.readAllBytes(Paths.get(localPdfPath))
24+
Base64.getEncoder.encodeToString(pdfBytes)
25+
}
26+
27+
private val fireworksModelPrefix = "accounts/fireworks/models/"
28+
override val service: OpenAIChatCompletionService = ChatCompletionProvider.fireworks
29+
30+
val messages: Seq[BaseMessage] = Seq(
31+
SystemMessage("You are a helpful assistant."),
32+
UserSeqMessage(
33+
Seq(
34+
TextContent("What are the candidate's BA and MBA GPAs?"),
35+
ImageURLContent(
36+
s"data:application/pdf;base64,${base64Pdf}#transform=inline"
37+
)
38+
)
39+
)
40+
)
41+
42+
override protected def run: Future[_] =
43+
service
44+
.createChatCompletion(
45+
messages,
46+
settings = CreateChatCompletionSettings(
47+
model =
48+
fireworksModelPrefix + NonOpenAIModelId.llama_v3p3_70b_instruct, // phi_3_vision_128k_instruct
49+
temperature = Some(0),
50+
max_tokens = Some(1000)
51+
)
52+
)
53+
.map(printMessageContent)
54+
}

0 commit comments

Comments
 (0)