DAOTracker Support (#380)

* init

* testing next

* tests

* remove brainstorming doc

* fix

* fix

* fix

* fixes

* fix

* :face_palm:

* fix

* fix

* timeout

* found it

* fix

* checking correct things

* see if dao tracker is even a thing

* fix datasource.yaml

* .toLowerCase()

* fix

* fix

* wait longer?

* feedback changes

* updates based on feedback

* fixes

* change to 31 so tests pass

* fixed

* fix lint

* new DAOtracker

* revert

* fix?

* fix

* ci pls work

* maybe this?

* DaoCreator only

* Hardcoded Blacklist

* revert

* fix lint

Author: dOrgJelli

README.md

@@ -72,7 +72,6 @@ In order to add support for a new contract follow these steps:
          list of entities that are written by the the mapping.
       3. [`eventHandlers`](https://github.com/graphprotocol/graph-node/blob/master/docs/subgraph-manifest.md#1522-eventhandler) -
          map of solidity event signatures to event handlers in mapping code.
-      4. [`templates`]([https://](https://github.com/graphprotocol/graph-node/blob/master/docs/subgraph-manifest.md#17-dynamicdatasource)) - list of datasource mappings that are created by the mapping.
    4. `test/integration/<contract name>.spec.ts`
 
 3. Add your contract to `ops/mappings.json`. Under the JSON object for the network your contract is located at, under the `"mappings"` JSON array, add the following.
@@ -107,6 +106,24 @@ In order to add support for a new contract follow these steps:
 
 To index a DAO please follow the instructions here: [https://github.com/daostack/subgraph/blob/master/documentations/Deployment.md#indexing-a-new-dao](https://github.com/daostack/subgraph/blob/master/documentations/Deployment.md#indexing-a-new-dao)
 
+## Add a new datasource template
+
+Datasource templates allow you to index blockchain data from addresses the subgraph finds out about at runtime. This is used to dynamically index newly deployed DAOs. To add a new contract ABI that can be used as a template within your mappings, modify the `ops/templates.json` file like so:
+
+```json
+{
+   "templates": [
+      ...,
+      {
+         "name": "<contract name as appears in `abis/arcVersion` folder>",
+         "mapping": "<name of the `src/mappings/...` folder to be used with this contract>",
+         "start_arcVersion": "<contract arc version under which the abi is located in the `abis` folder>",
+         "end_arcVersion": "(optional) <contract arc version under which the abi is located in the `abis` folder> if not given, all future versions of this `name`'s contract ABI will be added as a template for this mapping"
+      }
+   ]
+}
+```
+
 ## Deploy Subgraph
 
 To deploy the subgraph, please follow the instructions below:
@@ -148,3 +165,16 @@ The docker images are available as:
 
 `daostack/subgraph-postgres:${network}-${migration-version}-${subgraph-version}`
 `daostack/subgraph-ipfs:${network}-${migration-version}-${subgraph-version}`
+
+## Blacklist a malicious DAO
+Add the DAO's Avatar address to the `ops/blacklist.json` file in the proper network array. For example, blacklisting `0xF7074b67B4B7830694a6f58Df06375F00365d2c2` on mainnet would look like:
+```json
+{
+  "private": [],
+  "kovan": [],
+  "rinkeby": [],
+  "mainnet": [
+     "0xF7074b67B4B7830694a6f58Df06375F00365d2c2"
+  ]
+}
+```

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       ipfs: 'ipfs:5001'
       ethereum: 'private:http://ganache:8545'
       GRAPH_IPFS_TIMEOUT: '2'
-      GRAPH_MAX_IPFS_FILE_BYTES: '120000'
+      GRAPH_MAX_IPFS_FILE_BYTES: '200000'
       GRAPH_GRAPHQL_MAX_FIRST: '1000'
   ipfs:
     build: docker/ipfs

ops/blacklist.json

@@ -0,0 +1,6 @@
+{
+  "private": [],
+  "kovan": [],
+  "rinkeby": [],
+  "mainnet": []
+}

ops/generate-abis.js

@@ -1,11 +1,10 @@
 const fs = require("fs-extra");
-const path = require("path");
 
 /**
  * Fetch all abis from @daostack/migration into the `abis` folder.
  */
 async function generateAbis() {
-  fs.copy("node_modules/@daostack/migration/abis", "abis");
+  fs.copySync("node_modules/@daostack/migration/abis", `${__dirname}/../abis`);
 }
 
 if (require.main === module) {

ops/generate-contractsinfo.js

@@ -1,8 +1,8 @@
 const fs = require("fs");
-const yaml = require("js-yaml");
 const { migrationFileLocation: defaultMigrationFileLocation, network } = require("./settings");
+const { forEachTemplate } = require("./utils");
 const path = require("path");
-const currentDir = path.resolve(`${__dirname}`)
+const currentDir = path.resolve(`${__dirname}`);
 
 /**
  * Generate a `src/contractinfo.js` file from `migration.json`
@@ -20,8 +20,8 @@ async function generateContractInfo(opts={}) {
   const migration = JSON.parse(fs.readFileSync(require.resolve(opts.migrationFile), "utf-8"));
 
   let versions = migration[network].base
-  let buffer = "import { setContractInfo } from './utils';\n";
-  buffer += "// this code was generated automatically . please not edit it -:)\n";
+  let buffer = "import {\n  setBlacklistedDAO,\n  setContractInfo,\n  setTemplateInfo,\n} from './utils';\n";
+  buffer += "\n// this code was generated automatically . please not edit it -:)\n";
   buffer += "/* tslint:disable:max-line-length */\n";
 
   buffer += "export function setContractsInfo(): void {\n";
@@ -53,6 +53,26 @@ async function generateContractInfo(opts={}) {
     });
     buffer += "}\n";
 
+    buffer += "\nexport function setTemplatesInfo(): void {\n";
+
+    forEachTemplate((name, mapping, arcVersion) => {
+      const templateName = arcVersion.replace(/\.|-/g, '_');
+      buffer += `    setTemplateInfo('${name}', '${arcVersion}', '${name}_${templateName}');\n`;
+    });
+
+    buffer += "}\n";
+
+    const blacklist = require("./blacklist.json")[network];
+
+    buffer += "\nexport function setBlacklistedDAOs(): void {\n";
+
+    blacklist.forEach(function(avatar) {
+      buffer += `    setBlacklistedDAO('${avatar.toLowerCase()}');\n`;
+    });
+
+    buffer += "    return;\n";
+    buffer += "}\n";
+
     fs.writeFileSync(
       `${currentDir}/../src/contractsInfo.ts`,
       buffer,

ops/generate-subgraph.js

@@ -3,6 +3,7 @@ const path = require("path")
 const yaml = require("js-yaml");
 const { migrationFileLocation: defaultMigrationFileLocation,
 network ,startBlock} = require("./settings");
+const { versionToNum, forEachTemplate } = require("./utils");
 const mappings = require("./mappings.json")[network].mappings;
 const { subgraphLocation: defaultSubgraphLocation } = require('./graph-cli')
 
@@ -38,10 +39,13 @@ async function generateSubgraph(opts={}) {
     throw Error(`The following contracts are missing addresses: ${missing.toString()}`);
   }
 
+  const templates = buildTemplates();
+
   const subgraph = {
     specVersion: "0.0.1",
     schema: { file: "./schema.graphql" },
-    dataSources
+    dataSources,
+    templates
   };
 
   fs.writeFileSync(
@@ -55,19 +59,17 @@ function combineFragments(fragments, isTemplate, addresses, missingAddresses) {
   let ids = [];
   return fragments.map(mapping => {
     const contract = mapping.name;
+    const version = mapping.arcVersion;
     const fragment = `${__dirname}/../src/mappings/${mapping.mapping}/datasource.yaml`;
-    var abis, entities, eventHandlers, templates, file, yamlLoad, abi;
+    var abis, entities, eventHandlers, file, yamlLoad, abi;
 
     if (fs.existsSync(fragment)) {
       yamlLoad = yaml.safeLoad(fs.readFileSync(fragment, "utf-8"));
       file = `${__dirname}/../src/mappings/${mapping.mapping}/mapping.ts`;
       eventHandlers = yamlLoad.eventHandlers;
       entities = yamlLoad.entities;
-      templates = yamlLoad.templates;
       abis = (yamlLoad.abis || [contract]).map(contractName => {
-        const version = mapping.arcVersion;
-        const strlen = version.length
-        const versionNum = Number(version.slice(strlen - 2, strlen));
+        const versionNum = versionToNum(version);
 
         if ((versionNum < 24) && (contractName === "UGenericScheme")) {
           return {
@@ -95,17 +97,17 @@ function combineFragments(fragments, isTemplate, addresses, missingAddresses) {
       if (mapping.dao === 'address') {
         contractAddress = mapping.address;
       } else if (mapping.dao === 'organs') {
-        contractAddress = addresses[network].test[mapping.arcVersion][mapping.dao][mapping.contractName];
+        contractAddress = addresses[network].test[version][mapping.dao][mapping.contractName];
       } else {
-        contractAddress = addresses[network][mapping.dao][mapping.arcVersion][mapping.contractName];
+        contractAddress = addresses[network][mapping.dao][version][mapping.contractName];
       }
 
       // If the contract isn't predeployed
       if (!contractAddress) {
         // Keep track of contracts that're missing addresses.
         // These contracts should have addresses because they're not
         // templated contracts aren't used as templates
-        const daoContract = `${network}-${mapping.arcVersion}-${mapping.contractName}-${mapping.dao}`;
+        const daoContract = `${network}-${version}-${mapping.contractName}-${mapping.dao}`;
         if (missingAddresses[daoContract] === undefined) {
           missingAddresses[daoContract] = true;
         }
@@ -131,6 +133,12 @@ function combineFragments(fragments, isTemplate, addresses, missingAddresses) {
       return;
     }
 
+    if (isTemplate) {
+      // convert name to be version specific
+      const classVersion = version.replace(/\.|-/g, '_');
+      name = `${name}_${classVersion}`
+    }
+
     var result = {
       kind: 'ethereum/contract',
       name: `${name}`,
@@ -147,21 +155,31 @@ function combineFragments(fragments, isTemplate, addresses, missingAddresses) {
       }
     };
 
-    if (templates && templates.length) {
-      result.templates = combineFragments(
-        templates.map(template => ({
-          name: template,
-          mapping: template,
-          arcVersion: mapping.arcVersion
-        })),
-        true, addresses, missingAddresses
-      );
-    }
-
     return result;
   });
 }
 
+function buildTemplates() {
+  let results = [];
+
+  forEachTemplate((name, mapping, arcVersion) => {
+    results.push(
+      ...combineFragments(
+        [{
+          name,
+          mapping,
+          arcVersion
+        }],
+        true,
+        undefined,
+        undefined
+      )
+    );
+  });
+
+  return results;
+}
+
 if (require.main === module) {
   generateSubgraph().catch(err => {
     console.log(err);

ops/mappings.json

@@ -610,6 +610,13 @@
         "dao": "base",
         "mapping": "Redeemer",
         "arcVersion": "0.0.1-rc.32"
+      },
+      {
+        "name": "DAOTracker",
+        "contractName": "DAOTracker",
+        "dao": "base",
+        "mapping": "DAOTracker",
+        "arcVersion": "0.0.1-rc.32"
       }
     ]
   },
@@ -824,6 +831,13 @@
         "dao": "base",
         "mapping": "GenesisProtocol",
         "arcVersion": "0.0.1-rc.24"
+      },
+      {
+        "name": "DAOTracker",
+        "contractName": "DAOTracker",
+        "dao": "base",
+        "mapping": "DAOTracker",
+        "arcVersion": "0.0.1-rc.32"
       }
     ]
   },
@@ -969,6 +983,13 @@
       "dao": "base",
       "mapping": "DAORegistry",
       "arcVersion": "0.0.1-rc.30"
+    },
+    {
+      "name": "DAOTracker",
+      "contractName": "DAOTracker",
+      "dao": "base",
+      "mapping": "DAOTracker",
+      "arcVersion": "0.0.1-rc.32"
     }
   ]
   }

ops/templates.json

@@ -0,0 +1,24 @@
+{
+  "templates": [
+    {
+      "name": "Avatar",
+      "mapping": "Avatar",
+      "start_arcVersion": "0.0.1-rc.16"
+    },
+    {
+      "name": "Controller",
+      "mapping": "Controller",
+      "start_arcVersion": "0.0.1-rc.16"
+    },
+    {
+      "name": "DAOToken",
+      "mapping": "DAOToken",
+      "start_arcVersion": "0.0.1-rc.16"
+    },
+    {
+      "name": "Reputation",
+      "mapping": "Reputation",
+      "start_arcVersion": "0.0.1-rc.16"
+    }
+  ]
+}

ops/utils.js

@@ -0,0 +1,34 @@
+const fs = require("fs");
+
+function versionToNum(version) {
+  const strlen = version.length;
+  return Number(version.slice(strlen - 2, strlen));
+}
+
+function forEachTemplate(callback) {
+  const templates = require("./templates.json").templates;
+  const abiDirectories = fs.readdirSync(`${__dirname}/../abis`, { withFileTypes: true })
+        .filter(dirent => dirent.isDirectory())
+        .map(dirent => dirent.name);
+
+  for (var template of templates) {
+    const { name, mapping, start_arcVersion, end_arcVersion } = template;
+    const startNum = versionToNum(start_arcVersion);
+    const endNum = end_arcVersion ? versionToNum(end_arcVersion) : undefined;
+
+    // for each ABI directory, if it falls within start & end
+    for (var abiDirectory of abiDirectories) {
+      const arcVersion = abiDirectory;
+      const arcNum = versionToNum(arcVersion);
+
+      if (arcNum >= startNum && (endNum === undefined || arcNum <= endNum)) {
+        callback(name, mapping, arcVersion);
+      }
+    }
+  }
+}
+
+module.exports = {
+  versionToNum,
+  forEachTemplate
+}

package-lock.json

@@ -1,5 +1,5 @@
 import { Address, BigDecimal, BigInt, ByteArray, Bytes, crypto, Entity, store, Value} from '@graphprotocol/graph-ts';
-import { setContractsInfo } from '../contractsInfo';
+import { setContractsInfo, setTemplatesInfo } from '../contractsInfo';
 import {
   NewContributionProposal,
   ProposalExecuted,
@@ -208,6 +208,7 @@ export function handleRegisterScheme(avatar: Address,
   );
   if (isFirstRegister == null) {
     setContractsInfo();
+    setTemplatesInfo();
     let dao = daoModule.insertNewDAO(avatar, nativeTokenAddress , nativeReputationAddress);
     insertToken(hexToAddress(dao.nativeToken), avatar.toHex());
     insertReputation(

src/domain/index.ts

@@ -199,6 +199,11 @@ type ContractInfo @entity {
 	address: Bytes!
 }
 
+type TemplateInfo @entity {
+	id: ID!
+	templateName: String!
+}
+
 type Event @entity {
 	id: ID!
 	type: EventType!

src/domain/schema.graphql

@@ -0,0 +1,8 @@
+abis:
+  - DAOTracker
+entities:
+  - DAOTrackerContract
+  - BlacklistedDAO
+eventHandlers:
+  - event: TrackDAO(indexed address,address,address,address,address,string)
+    handler: handleTrackDAO