Hi @sopel and @AtlaBrio ,
Thanks for your feedback. Here’s a new version of the app that includes processing records in an asynchronous, paginated manner.
manifwest.yml
modules:
macro:
- key: forge-user-privacy-reporting-demo
function: macro-fn
title: Forge User Privacy Reporting Demo
description: Demonstrated how a Forge app should ensure it complies with user privacy guidelines.
consumer:
- key: report-pd-consumer
queue: report-pd-queue
resolver:
function: on-report-pd-fn
method: pd-report-listener
scheduledTrigger:
- key: personal-data-trigger
function: pd-trigger-fn
interval: hour
function:
- key: macro-fn
handler: index.renderMacro
- key: on-report-pd-fn
handler: index.onReportPDQueueData
- key: pd-trigger-fn
handler: index.reportPersonalDataTrigger
permissions:
scopes:
- storage:app
- report:personal-data
app:
id: ari:cloud:ecosystem::app/xxxxxxx
index.tsx
import ForgeUI, {
Button,
ButtonSet,
Fragment,
Heading,
Macro,
render,
Text,
useAction,
useState
} from '@forge/ui';
import {
privacy,
storage,
startsWith,
ListResult,
Result
} from '@forge/api';
import { Queue } from '@forge/events';
import Resolver from "@forge/resolver";
/*
* This app demonstrates how a Forge app can invoke the personal data reporting API to determine
* what personal data needs to be purged. The app provides a Confluence macro that allows the
* user to control the storing of personal data by the app and a button within the macro to force
* the reporting for testing purposes.
* In addition to providing the user with a manual control for forcing the reporting of personal
* data, the app also defines a scheduled trigger to demonstrate how an app would periodically
* report its personal data usage. The scheduled trigger fires once per how which is likely more
* often than a real app would need to do.
*
* How to use the app:
* (a) Install the app in Confluence.
* (b) Insert the app's macro, Forge User Privacy Reporting Demo, in a page.
* (c) Click the "Store active user" button to store personal details for an active user.
* (d) Click the "Store closed user" button to store personal details for a closed user.
* (e) Observe the macro shows two records are being stored.
* (f) Click the "Report personal data" button and observe the closed account details isremoved from
* the app's storage.
* (g) Click the "Store closed user" button to store personal details for a closed user again.
* (h) Wait for an hour or so and refresh the page to observe the closed user's account details has
* been purged after the app's scheduled trigger runs.
*/
// These account IDs are from https://developer.atlassian.com/cloud/confluence/user-privacy-developer-guide/#testing
const activeTestAccountId = '5be24ad8b1653240376955d2';
const closedTestAccountId = '5be24ba3f91c106033269289';
const accountRecordStorageKeyPrefix = 'account-item';
// This constant is from https://developer.atlassian.com/platform/forge/runtime-reference/privacy-api/
const maxAllowedRecordsToReportPerBatch = 90;
const oneDayInMillis = 24 * 60 * 60 * 1000;
// This constant defines how often the reporting will be done per account record.
// const personalDataReportingPeriodMillis = 7 * oneDayInMillis; // A realisitc value for an app to use.
const personalDataReportingPeriodMillis = 0; // A value to use for testing the app.
// This constant defines the maximum number of records it will attempt to report on per batch. If there
// are additional records to report on, then a request will be enqueued using the Async events API.
const maxRecordsToReportPerBatch = 1;
// The UI is not the primary focus of this example app, but it does provide rudimentary presentation
// of all the data in set sizes defined by this constant.
const maxRecordsForUI = 10;
interface AccountRecord {
storageKey: string
accountId: string
email: string
updatedAt: string
nextReportDueBy: string
}
interface AccountRecordsQueryResult {
accountRecords: AccountRecord[]
nextCursor: undefined | string
}
interface AccountReportingAsyncContext {
nextCursor: undefined | string
}
interface PersonalDataReportItem {
accountId: string
updatedAt: string
}
const pdReportingQueue = new Queue({ key: 'report-pd-queue' });
const pdReportingResolver = new Resolver();
const buildRecordStorageKey = (accountId: string): string => {
return `${accountRecordStorageKeyPrefix}-${accountId}`;
}
const getStoredAccountRecords = async (maxRecords: number, cursor: undefined | string): Promise<AccountRecordsQueryResult> => {
console.log(`Querying stored account recors with cursor ${cursor}...`);
const limit = Math.min(maxAllowedRecordsToReportPerBatch, maxRecords);
const queryResult = await storage.query()
.where('key', startsWith(accountRecordStorageKeyPrefix))
.limit(limit)
.cursor(cursor)
.getMany() as ListResult<AccountRecord>;
const nextCursor = queryResult.nextCursor;
const accountRecords = queryResult.results.map((accountRecordResult: Result<AccountRecord>) => {
return accountRecordResult.value;
});
const accountRecordsQueryResult: AccountRecordsQueryResult = {
accountRecords: accountRecords,
nextCursor: nextCursor
}
return accountRecordsQueryResult;
}
const storeAccountRecord = async (accountId: string, emailAddressPrefix: string): Promise<void> => {
const storageKey = `${buildRecordStorageKey(accountId)}`;
const now = new Date();
const nextReportDueTime = new Date(now.getTime() + personalDataReportingPeriodMillis);
const accountRecord: AccountRecord = {
storageKey: storageKey,
accountId: accountId,
email: `${emailAddressPrefix}@example-domain.com`,
updatedAt: now.toISOString(),
nextReportDueBy: nextReportDueTime.toISOString()
}
await storage.set(storageKey, accountRecord);
}
const deleteAccountRecordsWithId = async (accountId: string): Promise<void> => {
const accountStoragePrefix = buildRecordStorageKey(accountId);
const queryResult = await storage.query()
.where('key', startsWith(accountStoragePrefix))
.getMany() as ListResult<AccountRecord>;
for (const accountRecordResult of queryResult.results) {
console.log(` * deleting record ${accountRecordResult.value.storageKey}`);
await storage.delete(accountRecordResult.value.storageKey);
}
}
/**
* @returns true if the reporting is complete, false otherwise.
*/
const reportPersonalDataAsync = async (cursor: undefined | string): Promise<boolean> => {
let accountPurgeCount = 0;
const accountRecordsQueryResult = await getStoredAccountRecords(maxRecordsToReportPerBatch, cursor);
console.log(`Checking if any personal data needs to be cleaned up...`);
// NOTE: this does not handle arbitrary sizes of data because the Forge runtime may
// time out. For a robust implementation, use the Forge Async events API.
const payload: PersonalDataReportItem[] = [];
for (const accountRecord of accountRecordsQueryResult.accountRecords) {
console.log(`Analysing record ${JSON.stringify(accountRecord, null, 2)}...`);
const now = new Date().getTime();
const isReportDue = accountRecord.nextReportDueBy ? now > Date.parse(accountRecord.nextReportDueBy) : true;
if (isReportDue) {
const item: PersonalDataReportItem = {
accountId: accountRecord.accountId,
updatedAt: accountRecord.updatedAt
}
payload.push(item);
} else {
console.log(`Skipping the reporting of account ${accountRecord.accountId} since a report is not due until ${accountRecord.nextReportDueBy}.`);
}
}
console.log(`Built payload: ${JSON.stringify(payload, null, 2)}`);
// https://developer.atlassian.com/platform/forge/runtime-reference/privacy-api/
const updates = await privacy.reportPersonalData(payload);
if (updates && updates.length) {
console.log(`Some personal data may need to be purged from the app's data store:`);
for (const update of updates) {
if (update.status === 'closed') {
await deleteAccountRecordsWithId(update.accountId);
accountPurgeCount++;
} else if (update.status === 'updated') {
// The personal data that the app is storing may be stale. This app won't do
// anything in this case, but aniother app may like to delete the record or
// refresh the data.
}
console.log(` * purged ${accountPurgeCount} acount(s).`);
}
} else {
console.log(`No personal data needs to be purged from the app's data store for query with cursor ${cursor}.`);
}
const couldBeMoreRecordsToReportOn = accountRecordsQueryResult.nextCursor !== undefined;
if (couldBeMoreRecordsToReportOn) {
console.log(`Pushing PD reporting queue item with cursor ${accountRecordsQueryResult.nextCursor}...`);
const accountReportingAsyncContext: AccountReportingAsyncContext = {
nextCursor: accountRecordsQueryResult.nextCursor
}
await pdReportingQueue.push({
"accountReportingAsyncContext": accountReportingAsyncContext as any
});
}
const reportingComplete = !couldBeMoreRecordsToReportOn;
return reportingComplete;
}
pdReportingResolver.define("pd-report-listener", async (queueItem) => {
const accountReportingAsyncContext = queueItem.payload.accountReportingAsyncContext as AccountReportingAsyncContext;
console.log(`Received PD reporting queue item with cursor ${accountReportingAsyncContext.nextCursor}`);
await reportPersonalDataAsync(accountReportingAsyncContext.nextCursor);
});
export const onReportPDQueueData = pdReportingResolver.getDefinitions();
export const reportPersonalDataTrigger = async (): Promise<any> => {
const cursor = undefined;
const reportingComplete = await reportPersonalDataAsync(cursor);
const triggerResponse = {
body: `{"reportingComplete": ${reportingComplete}}`,
headers: {
'Content-Type': ['application/json'],
'X-Request-Id': [`rnd-${Math.random() }`]
},
statusCode: 200,
statusText: 'OK'
};
return triggerResponse;
}
const MacroApp = () => {
const [initialStoredAccountRecords] = useAction(value => value, async () => await getStoredAccountRecords(maxRecordsForUI, undefined));
const [accountRecordsQueryResult, setAccountRecordsQueryResult] = useState<AccountRecordsQueryResult>(initialStoredAccountRecords);
const onRefreshView = async (): Promise<void> => {
const cursor = undefined;
const accountRecordsQueryResult = await getStoredAccountRecords(maxRecordsForUI, cursor);
setAccountRecordsQueryResult(accountRecordsQueryResult);
}
const onViewNextSetOfRecords = async (): Promise<void> => {
const cursor = accountRecordsQueryResult.nextCursor;
const nextAccountRecordsQueryResult = await getStoredAccountRecords(maxRecordsForUI, cursor);
setAccountRecordsQueryResult(nextAccountRecordsQueryResult);
}
const onDeleteAccountRecords = async (): Promise<void> => {
await deleteAccountRecordsWithId(activeTestAccountId);
await deleteAccountRecordsWithId(closedTestAccountId);
await onRefreshView();
}
const onStoreActiveAccountRecords = async (): Promise<void> => {
await storeAccountRecord(activeTestAccountId, 'active-user');
await onRefreshView();
}
const onStoreClosedAccountRecords = async (): Promise<void> => {
await storeAccountRecord(closedTestAccountId, 'closed-user');
await onRefreshView();
}
const onReportPersonalData = async () => {
const cursor = undefined;
await reportPersonalDataAsync(cursor);
}
const renderStoredAccountRecords = () => {
if (accountRecordsQueryResult.accountRecords.length) {
return accountRecordsQueryResult.accountRecords.map((accountRecord: AccountRecord) => {
return (
<Text>* {accountRecord.accountId}: {accountRecord.email}</Text>
);
})
} else {
return (<Text>There is not any stored account data</Text>);
}
}
return (
<Fragment>
<Heading size="large">Forge user privacy reporting demo</Heading>
<Heading size="medium">Account data</Heading>
{renderStoredAccountRecords()}
<Heading size="medium">View operations</Heading>
<ButtonSet>
<Button text="Refresh" onClick={onRefreshView} />
<Button text="View next set of records" disabled={accountRecordsQueryResult.nextCursor === undefined} onClick={onViewNextSetOfRecords} />
</ButtonSet>
<Heading size="medium">Account operations</Heading>
<ButtonSet>
<Button text="Delete users" disabled={accountRecordsQueryResult.accountRecords.length === 0} onClick={onDeleteAccountRecords} />
<Button text="Store active users" onClick={onStoreActiveAccountRecords} />
<Button text="Store closed users" onClick={onStoreClosedAccountRecords} />
<Button text="Report personal data" onClick={onReportPersonalData} />
</ButtonSet>
</Fragment>
);
};
export const renderMacro = render(
<Macro
app={<MacroApp />}
/>
);
Please let me know if you spot any issues or if it is omitting important aspects relating to the reporting of personal data.
I’ve also included one or two tweaks such as avoiding over reporting.
Regards,
Dugald