diff --git a/gulpfile.js b/gulpfile.js index ce11d101..699daff9 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -229,6 +229,13 @@ gulp.task("bundleAppendIsInstalledMarker", function () { return merge(tasks); }); +gulp.task("bundleContentScript", function () { + var extensionRoot = PATHS.BUILDROOT + "scripts/extensions/"; + var files = ["contentScript.js"]; + var tasks = generateBrowserifyTasks(extensionRoot, files); + return merge(tasks); +}); + gulp.task("bundleClipperUI", function () { var extensionRoot = PATHS.BUILDROOT + "scripts/clipperUI/"; var files = ["clipper.js", "pageNav.js", "localeSpecificTasks.js", "unsupportedBrowser.js"]; @@ -299,6 +306,7 @@ gulp.task("bundleTests", function () { gulp.task("bundle", function(callback) { runSequence( "bundleAppendIsInstalledMarker", + "bundleContentScript", "bundleClipperUI", "bundleLogManager", "bundleBookmarklet", @@ -512,6 +520,10 @@ function exportChromeJS() { PATHS.BUNDLEROOT + "appendIsInstalledMarker.js" ]).pipe(concat("appendIsInstalledMarker.js")).pipe(gulp.dest(targetDir)); + var contentScriptTask = gulp.src([ + PATHS.BUNDLEROOT + "contentScript.js" + ]).pipe(concat("contentScript.js")).pipe(gulp.dest(targetDir)); + var chromeExtensionTask = gulp.src([ targetDir + "logManager.js", targetDir + "oneNoteApi.min.js", @@ -537,9 +549,9 @@ function exportChromeJS() { ]).pipe(concat("chromePageNavInject.js")).pipe(gulp.dest(targetDir)); if (commonTask) { - return merge(commonTask, appendIsInstalledMarkerTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); + return merge(commonTask, appendIsInstalledMarkerTask, contentScriptTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); } - return merge(chromeExtensionTask, appendIsInstalledMarkerTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); + return merge(chromeExtensionTask, appendIsInstalledMarkerTask, contentScriptTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); } function exportChromeCSS() { diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index e6c1b544..07ce7d81 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -2,6 +2,73 @@ import {ClientType} from "../../clientType"; import {WebExtension} from "../webExtensionBase/webExtension"; +function sendMessageToContentScript(tabs, msg) { + if (tabs.length > 0) { + chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) { + if (msg !== "GET_ALL_YOUTUBE_DETAILS") { + console.log(response.cMessage); + } + }); + } +} + +/****** START CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/ + +chrome.runtime.onMessage.addListener( + function (request, sender, sendResponse) { + // Received all details from content script, now send back to native application + sendMessageToNativeApplication({ + youtubeURL: request.youtubeURL, + documentTitle: request.documentTitle, + videoId: request.videoId, + streamPlayer: request.streamPlayer, + pictureOfVideo: request.pictureOfVideo, + transcript: request.transcript + }); + } +); + +/****** END CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/ + +/****** START CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ + +let hostName = 'com.microsoft.onenote.stickynotes'; +let port = chrome.runtime.connectNative(hostName); + +function sendMessageToNativeApplication(msg) { + port.postMessage(msg); +} + +port.onMessage.addListener((response) => { + console.log('Received from native application: ' + JSON.stringify(response)); + console.log('window.location.href as known to the current background script is: ' + window.location.href); + /****** START CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/ + /** + * This is a background script and cannot access the DOM of the current page. + * To access the DOM of the current page, we need to send a message to the + * content script. + */ + chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { + /* + console.log('Getting youtube url from content script...'); + sendMessageToContentScript(tabs, "GET_YOUTUBE_URL"); + console.log('Getting video id from content script...'); + sendMessageToContentScript(tabs, "GET_VIDEO_ID"); + console.log('Getting stream player from content script...'); + sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); + */ + console.log('Getting all details from content script and sending back to native application...'); + sendMessageToContentScript(tabs, "GET_ALL_YOUTUBE_DETAILS"); + }); + /****** END CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/ +}); + +port.onDisconnect.addListener(() => { + console.log('Disconnected'); +}); + +/****** END CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ + WebExtension.browser = chrome; let clipperBackground = new WebExtension(ClientType.ChromeExtension, { debugLoggingInjectUrl: "chromeDebugLoggingInject.js", diff --git a/src/scripts/extensions/chrome/manifest.json b/src/scripts/extensions/chrome/manifest.json index 30718585..593b79b5 100644 --- a/src/scripts/extensions/chrome/manifest.json +++ b/src/scripts/extensions/chrome/manifest.json @@ -13,6 +13,11 @@ "js": ["appendIsInstalledMarker.js"], "run_at": "document_start", "all_frames": true + }, { + "matches": [""], + "js": ["contentScript.js"], + "run_at": "document_end", + "all_frames": true }], "web_accessible_resources": [ @@ -27,7 +32,8 @@ "tabs", "webRequest", "storage", - "webNavigation" + "webNavigation", + "nativeMessaging" ], "content_security_policy": "script-src 'self'; object-src 'self'", diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts new file mode 100644 index 00000000..c46cbdf8 --- /dev/null +++ b/src/scripts/extensions/contentScript.ts @@ -0,0 +1,184 @@ +console.log("Content script loaded"); + +/****** START CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + +// Listen for a message from the background script +chrome.runtime.onMessage.addListener( + function (request, sender, sendResponse) { + if (window.self !== window.top) { + // We're in an iframe, ignore the message + return; + } + console.log(JSON.stringify(request)); + if (request === undefined) { + sendResponse({ cMessage: "Invalid request received by content script" }); + } + switch (request.bMessage) { + /* + case "GET_YOUTUBE_URL": + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL() }); + break; + case "GET_VIDEO_ID": + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getVideoId() }); + break; + case "GET_STREAM_PLAYER": + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML }); + break; + */ + case "GET_ALL_YOUTUBE_DETAILS": + if (!isYoutube()) { + chrome.runtime.sendMessage({ + youtubeURL: "NOT_YOUTUBE", + documentTitle: getDocumentTitle(), + pictureOfVideo: "NOT_YOUTUBE", + transcript: "NOT_YOUTUBE" + }); + } else { + getTranscriptAndReturnAllYoutubeDetails(); + } + break; + default: + sendResponse({ cMessage: "Invalid message received by content script" }); + } + }); + +/****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + +function isYoutube() { + return document.location.href.indexOf("youtube.com") !== -1; +} + +function getDocumentTitle() { + return document.title; +} + +function getStreamPlayer() { + return document.getElementsByClassName("video-stream")[0]; +} + +function getPictureOfVideo() { + const player = getStreamPlayer() as HTMLVideoElement; + var canvas = document.createElement("canvas"); + canvas.width = player.videoWidth; + canvas.height = player.videoHeight; + canvas.getContext("2d").drawImage(player, 0, 0, canvas.width, canvas.height); + + /****** START IMAGE COMPRESSION LOGIC ******/ + + // Create a new canvas element + const newCanvas = document.createElement('canvas'); + + // Set the dimensions of the new canvas to the desired size + newCanvas.width = canvas.width / 10; // Reduce the width + newCanvas.height = canvas.height / 10; // Reduce the height + + // Get the 2D context of the new canvas + const ctx = newCanvas.getContext('2d'); + + // Draw the image from the original canvas onto the new canvas + ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, newCanvas.width, newCanvas.height); + + /****** END IMAGE COMPRESSION LOGIC ******/ + + // Get the data URL of the image from the new canvas + const img = newCanvas.toDataURL("image/png"); + return img; +} + +function getTranscriptAroundCurrentTimestamp(fullTranscript) { + var player = getStreamPlayer() as HTMLVideoElement; + var time = player.currentTime; + + let result = ""; + + for (let i = 0; i < fullTranscript.length; i++) { + if (fullTranscript[i][0] < time - 15) { + continue; + } + if (fullTranscript[i][0] > time + 15) { + break; + } + // Include transcripts within 10 seconds of the current timestamp + result += fullTranscript[i][1] + " "; + } + + return result; +} + +function getTranscriptAndReturnAllYoutubeDetails() { + const injectedCode = ` +h2 = document.createElement('h2') +h2.id = 'ytTranscript' +h2.innerText = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks[0].baseUrl +document.getElementById('content').appendChild(h2) +`; + var script = document.createElement("script"); + script.textContent = injectedCode; + (document.head).appendChild(script); + var ytTranscript = document.getElementById('ytTranscript'); + var subsUrl; + if (ytTranscript) { + subsUrl = document.getElementById('ytTranscript').innerText; + let xhr = new XMLHttpRequest(); + xhr.open('GET', subsUrl); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + // Manipulate this code to get the timestamp as well + let xml = new DOMParser().parseFromString(xhr.responseText, 'text/xml'); + let textNodes = Array.prototype.slice.call(xml.getElementsByTagName('text')); + let subsText = [] + for (const t of textNodes) { + var obj = []; + const start = t.getAttribute('start'); + const duration = t.getAttribute('dur'); + obj.push(start) + obj.push(t.textContent) + subsText.push(obj) + } + console.log(JSON.stringify(subsText)); + chrome.runtime.sendMessage({ + youtubeURL: getYoutubeURL(), + documentTitle: getDocumentTitle(), + pictureOfVideo: getPictureOfVideo(), + transcript: getTranscriptAroundCurrentTimestamp(subsText) + }); + } + } + xhr.send(); + } +} + +function parseParams(href) { + const noHash = href.split("#")[0]; + const paramString = noHash.split("?")[1]; + const params = {}; + if (paramString) { + const paramsArray = paramString.split("&"); + for (const kv of paramsArray) { + const tmparr = kv.split("="); + params[tmparr[0]] = tmparr[1]; + } + } + return params; +} + +function getVideoId() { + if (window.location.pathname == "/watch") { + return parseParams(window.location.href)["v"]; + } else if (window.location.pathname.indexOf("/embed/") == 0) { + return window.location.pathname.substring("/embed/".length); + } else { + return null; + } +} + +function getYoutubeURL() { + var player = getStreamPlayer() as HTMLVideoElement; + var time = player.currentTime; + var path = document.location.href; + path = path.split("?")[0]; + path += "?v=" + getVideoId() + "&t=" + Math.floor(time) + "s"; + return path; +} + +console.log("Content script finished executing"); diff --git a/winManifest.json b/winManifest.json new file mode 100644 index 00000000..a94b0d6f --- /dev/null +++ b/winManifest.json @@ -0,0 +1,9 @@ +{ + "name": "com.microsoft.onenote.stickynotes", + "description": "Native host for OneNote Extension", + "path": "C:\\Program Files\\Microsoft Office\\root\\Office16\\onenote.exe", + "type": "stdio", + "allowed_origins": [ + "chrome-extension://gopaeppdaekmbdnampgofgnfdcfojdbn/" + ] + } \ No newline at end of file