Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
03-19-2025 03:52 AM
I was close 🙃
For opensearch.nodes, you should include https:
# OpenSearch connection details
host = dbutils.secrets.get(scope="opensearch", key="host")
port = dbutils.secrets.get(scope="opensearch", key="port")
os_url = f"https://{host}"
index = "index_name"
def save_to_open_search(df):
writer = (
df.write.format("opensearch")
.option("opensearch.nodes", os_url)
.option("opensearch.port", port)
.option("pushdown", "true")
.option("opensearch.batch.write.refresh", "false")
.option("opensearch.mapping.id", "id")
.option("opensearch.write.operation", "upsert")
.option("opensearch.aws.sigv4.enabled", "true")
.option("opensearch.aws.sigv4.region", "us-west-2")
.option("opensearch.nodes.resolve.hostname", "false")
.option("opensearch.nodes.wan.only", "true")
.option("opensearch.net.ssl", "true")
.option("opensearch.batch.size.entries", "5000")
.option("opensearch.batch.size.bytes", "10mb")
)
writer.mode("append").save(index)