Skip to content

Commit 0b0e98d

Browse files
odeimaizodeimaiz
authored
🎨 [Frontend] Use cursor paginated storage API (#7388)
1 parent 72ed036 commit 0b0e98d

File tree

6 files changed

+95
-75
lines changed

6 files changed

+95
-75
lines changed

‎services/static-webserver/client/source/class/osparc/announcement/Tracker.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ qx.Class.define("osparc.announcement.Tracker", {
2626
members: {
2727
__checkInterval: null,
2828

29-
checkAnnouncements: async function() {
29+
checkAnnouncements: function() {
3030
osparc.data.Resources.get("announcements")
3131
.then(announcements => {
3232
osparc.announcement.AnnouncementUIFactory.getInstance().setAnnouncementsData((announcements && announcements.length) ? announcements : []);

‎services/static-webserver/client/source/class/osparc/data/Resources.js

+8
Original file line numberDiff line numberDiff line change
@@ -1208,10 +1208,18 @@ qx.Class.define("osparc.data.Resources", {
12081208
method: "GET",
12091209
url: statics.API + "/storage/locations/{locationId}/paths?size=1000"
12101210
},
1211+
getDatasetsPage: {
1212+
method: "GET",
1213+
url: statics.API + "/storage/locations/{locationId}/paths?cursor={cursor}&size=1000"
1214+
},
12111215
getPaths: {
12121216
method: "GET",
12131217
url: statics.API + "/storage/locations/{locationId}/paths?file_filter={path}&size=1000"
12141218
},
1219+
getPathsPage: {
1220+
method: "GET",
1221+
url: statics.API + "/storage/locations/{locationId}/paths?file_filter={path}&cursor={cursor}&size=1000"
1222+
},
12151223
requestSize: {
12161224
method: "POST",
12171225
url: statics.API + "/storage/locations/0/paths/{pathId}:size"

‎services/static-webserver/client/source/class/osparc/file/FilesTree.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ qx.Class.define("osparc.file.FilesTree", {
4343
this.base(arguments, null, "label", "children");
4444

4545
this.set({
46-
openMode: "none",
46+
openMode: "dbltap",
4747
decorator: "no-border",
4848
font: "text-14",
4949
});
@@ -244,6 +244,7 @@ qx.Class.define("osparc.file.FilesTree", {
244244
configureItem: item => {
245245
item.addListener("changeOpen", e => {
246246
if (e.getData() && !item.getLoaded()) {
247+
item.setLoaded(true);
247248
const locationId = item.getLocation();
248249
const path = item.getPath();
249250
this.requestPathItems(locationId, path);

‎services/static-webserver/client/source/class/osparc/file/TreeFolderView.js

+12-18
Original file line numberDiff line numberDiff line change
@@ -101,34 +101,28 @@ qx.Class.define("osparc.file.TreeFolderView", {
101101
folderTree.addListener("selectionChanged", () => {
102102
const selectedModel = folderTree.getSelectedItem();
103103
if (selectedModel) {
104-
if (selectedModel.getPath() && !selectedModel.getLoaded()) {
105-
folderTree.requestPathItems(selectedModel.getLocation(), selectedModel.getPath())
106-
.then(pathModel => {
107-
if (osparc.file.FilesTree.isDir(pathModel)) {
108-
folderViewer.setFolder(pathModel);
109-
}
110-
});
111-
} else if (osparc.file.FilesTree.isDir(selectedModel)) {
104+
if (osparc.file.FilesTree.isDir(selectedModel)) {
112105
folderViewer.setFolder(selectedModel);
113106
}
107+
if (selectedModel.getPath() && !selectedModel.getLoaded()) {
108+
selectedModel.setLoaded(true);
109+
folderTree.requestPathItems(selectedModel.getLocation(), selectedModel.getPath());
110+
}
114111
}
115112
}, this);
116113

117114
folderViewer.addListener("openItemSelected", e => {
118115
const selectedModel = e.getData();
119116
if (selectedModel) {
120-
if (selectedModel.getPath() && !selectedModel.getLoaded()) {
121-
folderTree.requestPathItems(selectedModel.getLocation(), selectedModel.getPath())
122-
.then(pathModel => {
123-
folderTree.openNodeAndParents(pathModel);
124-
folderTree.setSelection(new qx.data.Array([pathModel]));
125-
if (osparc.file.FilesTree.isDir(pathModel)) {
126-
folderViewer.setFolder(pathModel);
127-
}
128-
});
129-
} else if (osparc.file.FilesTree.isDir(selectedModel)) {
117+
if (osparc.file.FilesTree.isDir(selectedModel)) {
130118
folderViewer.setFolder(selectedModel);
131119
}
120+
folderTree.openNodeAndParents(selectedModel);
121+
folderTree.setSelection(new qx.data.Array([selectedModel]));
122+
if (selectedModel.getPath() && !selectedModel.getLoaded()) {
123+
selectedModel.setLoaded(true);
124+
folderTree.requestPathItems(selectedModel.getLocation(), selectedModel.getPath());
125+
}
132126
}
133127
}, this);
134128

‎services/static-webserver/client/source/class/osparc/store/Data.js

+69-54
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,45 @@ qx.Class.define("osparc.store.Data", {
3535
"fileCopied": "qx.event.type.Data",
3636
},
3737

38+
statics: {
39+
getAllItems: async function(locationId, path, cursor, allItems = []) {
40+
if (allItems.length >= 10000) {
41+
const msg = qx.locale.Manager.tr("Oops... more than 10.000 items to be listed here. Maybe it's time to make a folder :).");
42+
osparc.FlashMessenger.logAs(msg, "WARNING");
43+
return allItems;
44+
}
45+
46+
const params = {
47+
url: {
48+
locationId,
49+
path: path || null,
50+
cursor: cursor || null,
51+
}
52+
};
53+
let pagResp = null;
54+
if (path) {
55+
pagResp = await osparc.data.Resources.fetch("storagePaths", cursor ? "getPathsPage" : "getPaths", params);
56+
} else {
57+
pagResp = await osparc.data.Resources.fetch("storagePaths", cursor ? "getDatasetsPage" : "getDatasets", params);
58+
}
59+
60+
let nextCursor = null;
61+
if (pagResp) {
62+
if (pagResp["items"]) {
63+
allItems.push(...pagResp["items"]);
64+
}
65+
if (pagResp["next_page"]) {
66+
nextCursor = pagResp["next_page"];
67+
}
68+
}
69+
70+
if (nextCursor) {
71+
return this.getAllItems(locationId, path, nextCursor, allItems);
72+
}
73+
return allItems;
74+
},
75+
},
76+
3877
members: {
3978
__locationsCached: null,
4079
__datasetsByLocationCached: null,
@@ -84,68 +123,44 @@ qx.Class.define("osparc.store.Data", {
84123
return null;
85124
},
86125

87-
getDatasetsByLocation: function(locationId) {
126+
getDatasetsByLocation: async function(locationId) {
88127
const data = {
89128
location: locationId,
90129
items: []
91130
};
92-
return new Promise((resolve, reject) => {
93-
if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) {
94-
reject(data);
95-
}
131+
if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) {
132+
return data;
133+
}
96134

97-
const cachedData = this.getDatasetsByLocationCached(locationId);
98-
if (cachedData) {
99-
resolve(cachedData);
100-
} else {
101-
const params = {
102-
url: {
103-
locationId
104-
}
105-
};
106-
osparc.data.Resources.fetch("storagePaths", "getDatasets", params)
107-
.then(pagResp => {
108-
if (pagResp["items"] && pagResp["items"].length>0) {
109-
data.items = pagResp["items"];
110-
}
111-
// Add it to cache
112-
this.__datasetsByLocationCached[locationId] = data.items;
113-
resolve(data);
114-
})
115-
.catch(err => {
116-
console.error(err);
117-
reject(data);
118-
});
119-
}
120-
});
135+
const cachedData = this.getDatasetsByLocationCached(locationId);
136+
if (cachedData) {
137+
return cachedData;
138+
}
139+
140+
try {
141+
const allItems = await this.self().getAllItems(locationId);
142+
this.__datasetsByLocationCached[locationId] = allItems;
143+
data["items"] = allItems;
144+
return data;
145+
} catch (err) {
146+
console.error(err);
147+
return data;
148+
}
121149
},
122150

123-
getItemsByLocationAndPath: function(locationId, path) {
124-
return new Promise((resolve, reject) => {
125-
// Get list of file meta data
126-
if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) {
127-
reject([]);
128-
}
151+
getItemsByLocationAndPath: async function(locationId, path) {
152+
// Get list of file meta data
153+
if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) {
154+
return [];
155+
}
129156

130-
const params = {
131-
url: {
132-
locationId,
133-
path,
134-
}
135-
};
136-
osparc.data.Resources.fetch("storagePaths", "getPaths", params)
137-
.then(pagResp => {
138-
if (pagResp["items"] && pagResp["items"].length>0) {
139-
resolve(pagResp["items"]);
140-
} else {
141-
resolve([]);
142-
}
143-
})
144-
.catch(err => {
145-
console.error(err);
146-
reject([]);
147-
});
148-
});
157+
try {
158+
const allItems = await this.self().getAllItems(locationId, path);
159+
return allItems;
160+
} catch (err) {
161+
console.error(err);
162+
return [];
163+
}
149164
},
150165

151166
getPresignedLink: function(download = true, locationId, fileUuid, fileSize) {

‎services/static-webserver/client/source/class/osparc/widget/StudyDataManager.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ qx.Class.define("osparc.widget.StudyDataManager", {
5858
if (!title) {
5959
title = osparc.product.Utils.getStudyAlias({firstUpperCase: true}) + qx.locale.Manager.tr(" Files");
6060
}
61-
return osparc.ui.window.Window.popUpInWindow(studyDataManager, title, osparc.dashboard.ResourceDetails.WIDTH, osparc.dashboard.ResourceDetails.HEIGHT);
61+
return osparc.ui.window.Window.popUpInWindow(studyDataManager, title, osparc.dashboard.ResourceDetails.WIDTH, osparc.dashboard.ResourceDetails.HEIGHT).set({
62+
maxHeight: document.documentElement.clientHeight,
63+
});
6264
},
6365
},
6466

0 commit comments

Comments
 (0)