jianmu
/
openobserve
mirror of https://github.com/openobserve/openobserve
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: implement ability to save search views (#1756) 

Created the standard GET,PUT,DELETE verbs for retrieving and
saving views which can be reused later.

Further details in:
Fixes https://github.com/openobserve/openobserve/issues/970
This commit is contained in:
Ankur Srivastava 2023-11-23 12:31:33 +01:00 committed by GitHub
parent 6401a9e075
commit c9008588f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 783 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/common/meta/mod.rs
View File

@ -14,7 +14,6 @@
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
pub mod alert;
pub mod common;
pub mod dashboards;
@ -27,6 +26,7 @@ pub mod middleware_data;
pub mod organization;
pub mod prom;
pub mod proxy;
pub mod saved_view;
pub mod search;
pub mod service;
pub mod sql;

79
src/common/meta/saved_view.rs Normal file
View File

@ -0,0 +1,79 @@
// Copyright 2023 Zinc Labs Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
#[derive(Serialize, Deserialize, ToSchema)]
pub struct CreateViewRequest {
/// Base64 encoded string, containing all the data for a given view.
/// This data is expected to be versioned so that the frontend can deserialize
/// as required.
pub data: serde_json::Value,
/// User-readable name of the view, doesn't need to be unique.
pub view_name: String,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct UpdateViewRequest {
/// Base64 encoded string, containing all the data for a given view.
/// This data is expected to be versioned so that the frontend can deserialize
/// as required.
pub data: serde_json::Value,
/// User-readable name of the view, doesn't need to be unique.
pub view_name: String,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct View {
pub org_id: String,
pub data: serde_json::Value,
pub view_id: String,
pub view_name: String,
}
/// Save the bandwidth for a given view, without sending the actual data
/// This is expected to be used for listing views.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct ViewWithoutData {
pub org_id: String,
pub view_id: String,
pub view_name: String,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct ViewsWithoutData {
pub views: Vec<ViewWithoutData>,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Views {
pub views: Vec<View>,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct DeleteViewResponse {
pub org_id: String,
pub view_id: String,
//TODO(ansrivas): Check if we have access to view_name
// pub view_name: String,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct CreateViewResponse {
pub org_id: String,
pub view_id: String,
pub view_name: String,
}

1
src/handler/http/request/search/mod.rs
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod saved_view;
use actix_web::{get, http::StatusCode, post, web, HttpRequest, HttpResponse};
use ahash::AHashMap;
use chrono::Duration;

252
src/handler/http/request/search/saved_view.rs
View File

@ -1,6 +1,246 @@
/// POST: to save a view
/// GET:
/// - List saved views
/// - Get one view
/// DELETE: saved views
/// PUT: Edit views
// Copyright 2023 Zinc Labs Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2023 Zinc Labs Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::meta::{
http::HttpResponse as MetaHttpResponse,
saved_view::{CreateViewRequest, UpdateViewRequest, CreateViewResponse, DeleteViewResponse},
};
use actix_web::{delete, get, post, put, web, HttpRequest, HttpResponse};
use std::io::Error;
use crate::service::db::saved_view;
// GetSavedView
//
// Retrieve a single saved view associated with this org.
//
#[utoipa::path(
context_path = "/api",
tag = "Saved Views",
operation_id = "GetSavedView",
security(
("Authorization"= [])
),
params(
("org_id" = String, Path, description = "Organization name"),
("view_id" = String, Path, description = "The view_id which was stored"),
),
responses(
(status = 200, description="Success", content_type = "application/json", body = View, example = json!({
"org_id": "some-org-id",
"view_id": "some-uuid-v4",
"view_name": "view-name",
"payload": "base64-encoded-object-as-sent-by-frontend"
})),
(status = 400, description="Failure", content_type = "application/json", body = HttpResponse),
(status = 500, description="Failure", content_type = "application/json", body = HttpResponse),
)
)]
#[get("/{org_id}/savedviews/{view_id}")]
pub async fn get_view(
path: web::Path<(String, String)>,
_req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, view_id) = path.into_inner();
let view = saved_view::get_view(&org_id, &view_id).await.unwrap();
Ok(MetaHttpResponse::json(view))
}
// ListSavedViews
//
// Retrieve the list of saved views.
//
#[utoipa::path(
context_path = "/api",
tag = "Saved Views",
operation_id = "ListSavedViews",
security(
("Authorization"= [])
),
params(
("org_id" = String, Path, description = "Organization name"),
),
responses(
(status = 200, description="Success", content_type = "application/json", body = Views, example = json!([{
"org_id": "some-org-id",
"view_name": "view-name",
"view_id": "view-id",
"payload": "base-64-encoded-versioned-payload"
}])),
(status = 400, description="Failure", content_type = "application/json", body = HttpResponse),
(status = 500, description="Failure", content_type = "application/json", body = HttpResponse),
)
)]
#[get("/{org_id}/savedviews")]
pub async fn get_views(path: web::Path<String>, _req: HttpRequest) -> Result<HttpResponse, Error> {
let org_id = path.into_inner();
let views = saved_view::get_views_list_only(&org_id).await.unwrap();
Ok(MetaHttpResponse::json(views))
}
// DeleteSavedViews
//
// Delete a view associated with this given org.
//
#[utoipa::path(
context_path = "/api",
tag = "Saved Views",
operation_id = "DeleteSavedViews",
security(
("Authorization"= [])
),
params(
("org_id" = String, Path, description = "Organization name"),
("view_id" = String, Path, description = "The view_id to delete"),
),
responses(
(status = 200, description="Success", content_type = "application/json", body = ResponseDeleteView, example = json!([{
"org_id": "some-org-id",
"view_id": "view_id",
}])),
(status = 400, description="Failure", content_type = "application/json", body = HttpResponse),
(status = 500, description="Failure", content_type = "application/json", body = HttpResponse),
)
)]
#[delete("/{org_id}/savedviews/{view_id}")]
pub async fn delete_view(
path: web::Path<(String, String)>,
_req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, view_id) = path.into_inner();
saved_view::delete_view(&org_id, &view_id).await.unwrap();
let resp = DeleteViewResponse { org_id, view_id };
Ok(MetaHttpResponse::json(resp))
}
// CreateSavedViews
//
// Create a view for later retrieval associated with the given search.
//
#[utoipa::path(
context_path = "/api",
tag = "Saved Views",
operation_id = "CreateSavedViews",
security(
("Authorization"= [])
),
params(
("org_id" = String, Path, description = "Organization name"),
),
request_body(content = RequestCreateView, description = "Create view data", content_type = "application/json"),
responses(
(status = 200, description="Success", content_type = "application/json", body = ResponseCreateView, example = json!([{
"org_id": "some-org-id",
"view_id": "view_id",
}])),
(status = 400, description="Failure", content_type = "application/json", body = HttpResponse),
(status = 500, description="Failure", content_type = "application/json", body = HttpResponse),
)
)]
#[post("/{org_id}/savedviews")]
pub async fn create_view(
path: web::Path<String>,
view: web::Json<CreateViewRequest>,
_req: HttpRequest,
) -> Result<HttpResponse, Error> {
let org_id = path.into_inner();
let created_view = saved_view::set_view(&org_id, &view).await.unwrap();
let resp = CreateViewResponse {
org_id,
view_id: created_view.view_id,
view_name: view.view_name.clone(),
};
Ok(MetaHttpResponse::json(resp))
}
// UpdateSavedViews
//
// Update a saved view
//
#[utoipa::path(
context_path = "/api",
tag = "Saved Views",
operation_id = "UpdateSavedViews",
security(
("Authorization"= [])
),
params(
("org_id" = String, Path, description = "Organization name"),
("view_id" = String, Path, description = "View id to be updated"),
),
request_body(content = RequestUpdateView, description = "Update view data", content_type = "application/json"),
responses(
(status = 200, description="Success", content_type = "application/json", body = View, example = json!([{
"org_id": "some-org-id",
"view_name": "view-name",
"view_id": "view-id",
"payload": "base-64-encoded-versioned-payload"
}])),
(status = 400, description="Failure", content_type = "application/json", body = HttpResponse),
(status = 500, description="Failure", content_type = "application/json", body = HttpResponse),
)
)]
#[put("/{org_id}/savedviews/{view_id}")]
pub async fn update_view(
path: web::Path<(String, String)>,
view: web::Json<UpdateViewRequest>,
_req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, view_id) = path.into_inner();
let updated_view = saved_view::update_view(&org_id, &view_id, &view)
.await
.unwrap();
Ok(MetaHttpResponse::json(updated_view))
}
#[cfg(test)]
mod tests {
use actix_web::{test, App};
use super::*;
#[actix_web::test]
async fn test_create_view_post() {
let payload = CreateViewRequest {
data: "base64-encoded-data".into(),
view_name: "query-for-blah".into(),
};
let app = test::init_service(App::new().service(create_view)).await;
let req = test::TestRequest::post()
.uri("/default/savedviews")
.set_json(&payload)
.to_request();
let resp = test::call_service(&app, req).await;
assert!(resp.status().is_success());
let json_body: CreateViewResponse = test::read_body_json(resp).await;
assert!(!json_body.view_id.is_empty());
}
}

5
src/handler/http/router/mod.rs
View File

@ -171,6 +171,11 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) {
.service(search::search)
.service(search::around)
.service(search::values)
.service(search::saved_view::create_view)
.service(search::saved_view::update_view)
.service(search::saved_view::get_view)
.service(search::saved_view::get_views)
.service(search::saved_view::delete_view)
.service(stream::schema)
.service(stream::settings)
.service(stream::delete_fields)

12
src/handler/http/router/openapi.rs
View File

@ -47,6 +47,11 @@ use crate::handler::http::request;
request::search::search,
request::search::around,
request::search::values,
request::search::saved_view::create_view,
request::search::saved_view::delete_view,
request::search::saved_view::get_view,
request::search::saved_view::get_views,
request::search::saved_view::update_view,
request::functions::list_functions,
request::functions::update_function,
request::functions::save_function,
@ -140,6 +145,12 @@ use crate::handler::http::request;
meta::search::RequestEncoding,
meta::search::Response,
meta::search::ResponseTook,
meta::saved_view::View,
meta::saved_view::Views,
meta::saved_view::CreateViewRequest,
meta::saved_view::DeleteViewResponse,
meta::saved_view::CreateViewResponse,
meta::saved_view::UpdateViewRequest,
meta::alert::Alert,
meta::alert::AlertList,
meta::alert::Condition,
@ -191,6 +202,7 @@ use crate::handler::http::request;
(name = "Logs", description = "Logs data ingestion operations"),
(name = "Dashboards", description = "Dashboard operations"),
(name = "Search", description = "Search/Query operations"),
(name = "Saved Views", description = "Collection of saved search views for easy retrieval"),
(name = "Alerts", description = "Alerts retrieval & management operations"),
(name = "Functions", description = "Functions retrieval & management operations"),
(name = "Organizations", description = "Organizations retrieval & management operations"),

1
src/service/db/mod.rs
View File

@ -23,6 +23,7 @@ pub mod functions;
pub mod kv;
pub mod metrics;
pub mod organization;
pub mod saved_view;
pub mod schema;
pub mod syslog;
pub mod triggers;

114
src/service/db/saved_view.rs Normal file
View File

@ -0,0 +1,114 @@
// Copyright 2023 Zinc Labs Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::{
infra::{db as infra_db, errors::Error},
meta::saved_view::{
CreateViewRequest, UpdateViewRequest, View, ViewWithoutData, Views, ViewsWithoutData,
},
utils::json,
};
pub const SAVED_VIEWS_KEY_PREFIX: &str = "/organization/savedviews";
pub async fn set_view(org_id: &str, view: &CreateViewRequest) -> Result<View, Error> {
let db = &infra_db::get_db().await;
let view_id = uuid::Uuid::new_v4().to_string();
let view = View {
org_id: org_id.into(),
view_id: view_id.clone(),
data: view.data.clone(),
view_name: view.view_name.clone(),
};
let key = format!("{}/{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id, view_id);
db.put(
&key,
json::to_vec(&view).unwrap().into(),
infra_db::NO_NEED_WATCH,
)
.await?;
Ok(view)
}
/// Update the given view
pub async fn update_view(
org_id: &str,
view_id: &str,
view: &UpdateViewRequest,
) -> Result<View, Error> {
let db = &infra_db::get_db().await;
let key = format!("{}/{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id, view_id);
let updated_view = match get_view(org_id, view_id).await {
Ok(original_view) => View {
data: view.data.clone(),
view_name: view.view_name.clone(),
..original_view
},
Err(e) => return Err(e),
};
db.put(
&key,
json::to_vec(&updated_view).unwrap().into(),
infra_db::NO_NEED_WATCH,
)
.await?;
Ok(updated_view)
}
/// Get the saved view id associated with an org_id
pub async fn get_view(org_id: &str, view_id: &str) -> Result<View, Error> {
let db = &infra_db::get_db().await;
let key = format!("{}/{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id, view_id);
let ret = db.get(&key).await?;
let view = json::from_slice(&ret).unwrap();
Ok(view)
}
/// Return all the saved views associated with a provided org_id
pub async fn get_views(org_id: &str) -> Result<Views, Error> {
let db = &infra_db::get_db().await;
let key = format!("{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id);
let ret = db.list_values(&key).await?;
let views: Vec<View> = ret
.iter()
.map(|view| json::from_slice(view).unwrap())
.collect();
Ok(Views { views })
}
/// Return all the saved views but query limited data only, associated with a provided org_id
/// This will not contain the payload.
pub async fn get_views_list_only(org_id: &str) -> Result<ViewsWithoutData, Error> {
let db = &infra_db::get_db().await;
let key = format!("{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id);
let ret = db.list_values(&key).await?;
let mut views: Vec<ViewWithoutData> = ret
.iter()
.map(|view| json::from_slice(view).unwrap())
.collect();
views.sort_by_key(|v| v.view_name.clone());
Ok(ViewsWithoutData { views })
}
/// Delete a saved view id associated with an org-id
// pub async fn delete_view(org_id: &str, view_id: &str) -> Result<View, Error> {
pub async fn delete_view(org_id: &str, view_id: &str) -> Result<(), Error> {
let db = &infra_db::get_db().await;
let key = format!("{}/{}/{}", SAVED_VIEWS_KEY_PREFIX, org_id, view_id);
db.delete(&key, false, infra_db::NO_NEED_WATCH).await?;
Ok(())
}

324
tests/saveviews.json Normal file
View File

@ -0,0 +1,324 @@
{
"data": {
"organizationIdetifier": "default",
"runQuery": false,
"loading": false,
"config": {
"splitterModel": 20,
"lastSplitterPosition": 0,
"splitterLimit": [
0,
40
],
"fnSplitterModel": 99.5,
"fnLastSplitterPosition": 0,
"fnSplitterLimit": [
40,
100
],
"refreshTimes": [
[
{
"label": "5 sec",
"value": 5
},
{
"label": "1 min",
"value": 60
},
{
"label": "1 hr",
"value": 3600
}
],
[
{
"label": "10 sec",
"value": 10
},
{
"label": "5 min",
"value": 300
},
{
"label": "2 hr",
"value": 7200
}
],
[
{
"label": "15 sec",
"value": 15
},
{
"label": "15 min",
"value": 900
},
{
"label": "1 day",
"value": 86400
}
],
[
{
"label": "30 sec",
"value": 30
},
{
"label": "30 min",
"value": 1800
}
]
]
},
"meta": {
"refreshInterval": "0",
"refreshIntervalLabel": "Off",
"showFields": true,
"showQuery": true,
"showHistogram": true,
"showDetailTab": false,
"toggleFunction": false,
"toggleSourceWrap": false,
"histogramDirtyFlag": false,
"sqlMode": false,
"queryEditorPlaceholderFlag": true,
"functionEditorPlaceholderFlag": true,
"resultGrid": {
"wrapCells": false,
"manualRemoveFields": true,
"rowsPerPage": 150,
"chartInterval": "10 second",
"chartKeyFormat": "HH:mm:ss",
"navigation": {
"currentRowIndex": 0
}
},
"flagWrapContent": false
},
"data": {
"query": "",
"parsedQuery": {},
"errorMsg": "",
"errorCode": 0,
"additionalErrorMsg": "",
"stream": {
"selectedStream": {
"label": "stream1",
"value": "stream1"
},
"selectedStreamFields": [
{
"name": "kubernetes_labels_pod_template_hash",
"ftsKey": false
},
{
"name": "kubernetes_container_name",
"ftsKey": false
},
{
"name": "level",
"ftsKey": false
},
{
"name": "kubernetes_labels_component",
"ftsKey": false
},
{
"name": "kubernetes_labels_operator_prometheus_io_shard",
"ftsKey": false
},
{
"name": "kubernetes_docker_id",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_instance",
"ftsKey": false
},
{
"name": "kubernetes_labels_prometheus",
"ftsKey": false
},
{
"name": "kubernetes_labels_statefulset_kubernetes_io_pod_name",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_part_of",
"ftsKey": false
},
{
"name": "kubernetes_pod_name",
"ftsKey": false
},
{
"name": "_timestamp",
"ftsKey": false
},
{
"name": "kubernetes_labels_role",
"ftsKey": false
},
{
"name": "kubernetes_labels_deploy",
"ftsKey": false
},
{
"name": "kubernetes_labels_controller_revision_hash",
"ftsKey": false
},
{
"name": "log",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_name",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_managed_by",
"ftsKey": false
},
{
"name": "kubernetes_namespace_name",
"ftsKey": false
},
{
"name": "method",
"ftsKey": false
},
{
"name": "kubernetes_host",
"ftsKey": false
},
{
"name": "kubernetes_annotations_kubectl_kubernetes_io_default_container",
"ftsKey": false
},
{
"name": "stream",
"ftsKey": false
},
{
"name": "kubernetes_annotations_prometheus_io_path",
"ftsKey": false
},
{
"name": "kubernetes_annotations_prometheus_io_scrape",
"ftsKey": false
},
{
"name": "message",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_component",
"ftsKey": false
},
{
"name": "kubernetes_pod_id",
"ftsKey": false
},
{
"name": "kubernetes_annotations_kubernetes_io_psp",
"ftsKey": false
},
{
"name": "kubernetes_annotations_prometheus_io_port",
"ftsKey": false
},
{
"name": "took",
"ftsKey": false
},
{
"name": "kubernetes_labels_name",
"ftsKey": false
},
{
"name": "kubernetes_labels_operator_prometheus_io_name",
"ftsKey": false
},
{
"name": "kubernetes_labels_app",
"ftsKey": false
},
{
"name": "kubernetes_labels_app_kubernetes_io_version",
"ftsKey": false
},
{
"name": "kubernetes_container_hash",
"ftsKey": false
},
{
"name": "code",
"ftsKey": false
},
{
"name": "kubernetes_container_image",
"ftsKey": false
}
],
"selectedFields": [
"kubernetes_docker_id",
"kubernetes_labels_component",
"level"
],
"filterField": "",
"addToFilter": "",
"streamType": "logs"
},
"resultGrid": {
"currentDateTime": "2023-11-21T13:29:01.550Z",
"currentPage": 0,
"columns": [
{
"name": "@timestamp",
"label": "timestamp (Asia/Calcutta)",
"align": "left",
"sortable": true
},
{
"name": "kubernetes_docker_id",
"label": "kubernetes_docker_id",
"align": "left",
"sortable": true,
"closable": true
},
{
"name": "kubernetes_labels_component",
"label": "kubernetes_labels_component",
"align": "left",
"sortable": true,
"closable": true
},
{
"name": "level",
"label": "level",
"align": "left",
"sortable": true,
"closable": true
}
]
},
"editorValue": "",
"datetime": {
"startTime": 1700573045992000,
"endTime": 1700573345992000,
"relativeTimePeriod": "5m",
"type": "relative"
},
"searchAround": {
"indexTimestamp": -1,
"size": 0,
"histogramHide": false
},
"tempFunctionName": "",
"tempFunctionContent": "",
"tempFunctionLoading": false
}
},
"view_name": "view1"
}