Smart adding of musical groups to Google Sheets via VK API, Tampermonkey and Telegram bot

How do I organize music storage in Google Sheets with support from multiple programs?


Given : Accounts in Google, VK, Telegram.
Task : Create a repository of music groups with the possibility of convenient replenishment.
Tools : Google Apps Script (GAS), VK API, Tampermonkey, Telegram Bot API.

Create Google Sheets


Create a new Google Sheets.

We make a fixed header (then all the cycles of enumerating cells, because of this, will begin not from 0, but from 1).

In principle, our template is ready.

The only thing is, I did not bind to the columns through the names (I did not want to use loops), so throughout the script I will refer to the hard-wired column number.

Click "Tools - Script Editor."



2 Create a script: Kod.gs and Bot.gs .

Code.gs
access_token = '1111111111' // VK token
ss = SpreadsheetApp.getActiveSpreadsheet() // 
sheet = ss.getSheets()[getNumSheet('')] // 
data = sheet.getDataRange().getValues() //  
numRows = sheet.getLastRow()+1 //  
faveTag = '.' //    

//   get 
function doGet(e) {
try {
    var link = e.parameters['link']
    /*
      A - 
      B - English
      C - 
      D - 
      E - 
      F - 
      G - 
    */
    var n = searchExists(link)
    if (n != false) return ContentService.createTextOutput(' '+n+'  !')
    //   API VK https://vk.com/dev/groups.getById
    var html = UrlFetchApp.fetch('https://api.vk.com/method/groups.getById.json?group_id='+link+'&fields=city,description&access_token='+access_token+'&v=5.107')
    //   json   response
    var json = JSON.parse(html).response[0]
    addInTableFromArray(json, link)
    sort()
    return ContentService.createTextOutput('')
  } catch (e) {
     return ContentService.createTextOutput('   doGet! '+e)
  }
}

//  /     
function searchGenre(txt) {
  var t = txt.match(/(?:(?:[||||]|genre|[|||]?)[\s:-]+){1}([a-z-\/-]+)/i)
  if(txt != '' && t) return t[1]
}

//     
function isRus(txt) {
  return txt.search(/[^A-Za-z0-9\/.:_]+/i)
}

// 
function sort(){
  //         
  var tableRange = "A2:G"+numRows
  var editedCell = sheet.getActiveCell()
  var range = sheet.getRange(tableRange)
  //     .    0,   1
  range.sort([{ 
    column : 4,
    ascending: true
  },{
    column: 1,
    ascending: true
  },{
    column: 2,
    ascending: true
  }])
}

//   
function searchExists(t) {
  for (var i = 1; i < data.length; i++) {
    if (t == data[i][2] || 'club'+t == data[i][2]){
      return data[i][0]+data[i][1]
    }
  }
  return false
}

//      
function addInfo(isBot = false) {
  var arr = []
  var j = 0
  var part = 1000 // - ,      . -      UrlFetchApp.fetch
  arr[j] = new Array()
  for (var i = 1; i < data.length; i++) {
    var txt = data[i][2].replace('https://vk.com/','').replace('vk.com/','').replace('^club','')
    if (txt != '' && data[i][0] == '' && data[i][1] == '' && data[i][3] == '' && data[i][4] == '' && data[i][5] == '' && data[i][6] == ''){
      arr[j].push(txt)
      if (arr[j].toString().length > part){
        j++
        arr[j] = new Array()
      }
    }
  }
  if (arr[0].length == 0){
    if (isBot) return false
    else{
      SpreadsheetApp.getUi().alert(' ,   ')
      return false
    }
  }
  //   
  for (var t = 0; t < Math.ceil(arr.toString().length/part); t++) {
    var html = UrlFetchApp.fetch('https://api.vk.com/method/groups.getById.json?group_ids='+arr[t].toString()+'&fields=city,description&access_token='+access_token+'&v=5.107')
    var json = JSON.parse(html).response
    if(json){
      for (var i = 0; i < json.length; i++) {
        var id = json[i].id
        var link = json[i].screen_name
        var name = json[i].name
        var description = (json[i].description)?json[i].description:''
        var city = (json[i].city)?json[i].city.title:''
        var rus = (isRus(name) != -1)?"A":"B"    
        //         arr
        for (var j = 1; j < data.length; j++) {
          var nameCell = data[j][2].replace('https://vk.com/','').replace('vk.com/','').replace('^club','')
          if (nameCell == link || nameCell == id){
            var num = j+1
            break
          }
        }
        sheet.getRange(rus+num).setValue(name.replace('=',''))
        sheet.getRange("C"+num).setValue('=HYPERLINK("https://vk.com/'+link+'";"'+link+'")')
        sheet.getRange("D"+num).setValue(city)
        sheet.getRange("E"+num).setValue(searchGenre(description))
        sheet.getRange("F"+num).setValue(description)
      }
    }
  }
  sort()
  return true
}

//     
function getVkFave(isBot = false) {
  var idTag
  var getTags = UrlFetchApp.fetch('https://api.vk.com/method/fave.getTags.json?access_token='+access_token+'&v=5.107')
  var res = JSON.parse(getTags).response
  var iTag = res.items
  for (var i = 0; i < iTag.length; i++) {
    if (iTag[i].name == faveTag) idTag = iTag[i].id
  }
  
  // 
  if (!isBot){
    var ui = SpreadsheetApp.getUi()
    var resp = ui.alert('     "'+faveTag+'"     . ?', ui.ButtonSet.YES_NO)
  }
  var inside = function (){
    var getPages = UrlFetchApp.fetch('https://api.vk.com/method/fave.getPages.json?tag_id='+idTag+'&fields=city,description&access_token='+access_token+'&v=5.107')
    var iPage = JSON.parse(getPages).response.items
    //    .   ,   
    for (var j = 0; j < iPage.length; j++) {
      var gr = iPage[j].group
      addInTableFromArray(gr)
      numRows++
        UrlFetchApp.fetch('https://api.vk.com/method/fave.removePage?group_id='+gr.id+'&access_token='+access_token+'&v=5.107')
        Utilities.sleep(1000) // 
    }
    sort()
  }
  if (isBot) { // ,      ui
    inside()
  }else if(resp == ui.Button.YES) {
    inside()
  }
}

//    
function addInTableFromArray(arr, linkIn) {
  if (linkIn){
    var link = linkIn
  }else{
    var link = arr.screen_name
    if (searchExists(link) != false) return false
  }
  var name = arr.name // 
  var description = (arr.description)?arr.description:'' //
  var city = (arr.city)?arr.city.title:'' // 
  //      ,      "English"
  var rus = (isRus(name) != -1)?"A":"B"
  //    
  sheet.getRange(rus+numRows).setValue(name.replace('=',''))
  sheet.getRange("C"+numRows).setValue('=HYPERLINK("https://vk.com/'+link+'";"'+link+'")')
  sheet.getRange("D"+numRows).setValue(city)
  sheet.getRange("E"+numRows).setValue(searchGenre(description))
  sheet.getRange("F"+numRows).setValue(description)
}
//  . .
function checkActualLink(){
  for (var i = 1; i < data.length; i++) {
    try {
      var num = i+1
      UrlFetchApp.fetch("https://vk.com/"+data[i][2])
      sheet.getRange('C'+num).setBackgroundColor('')
    }catch(err) {
      var num = i+1
      sheet.getRange('C'+num).setBackgroundColor('red')
    }
  }
}
//  URL  .     
function getStatusUrl(url){
   var options = {
     'muteHttpExceptions': true,
     'followRedirects': false
   }
   var response = UrlFetchApp.fetch(url.trim(), options)
   return response.getResponseCode()
}
//    
function getNumSheet(nameList){
  var s = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(nameList)
  if (s != null) {
    return s.getIndex()-1
  }else{
    return false
  }
}
// 
function escapeHtml(text) {
  return text
      .replace(/&/g, "")
      .replace(/</g, "")
      .replace(/>/g, "");
}
// 
function onOpen() {
  SpreadsheetApp.getUi()
  .createMenu(' ')
  .addItem('', 'sort')
  .addItem('   ', 'getVkFave')
  .addItem(' ', 'addInfo')
  .addToUi();
}

//   
function ll(t){
  return Logger.log(t)
}


Bot.gs
var botApi = 'https://api.telegram.org/bot1123123:AAA/' //  Telegram  
/* :
 /start -   
 /sort - 
 /getvkfave -     
 /addinfo -  
 /getall -   
*/
function doPost(e){
  var inp = JSON.parse(e.postData.contents)
  // 
  var sheet2 = ss.getSheets()[getNumSheet('')]
  sheet2.getRange('A1').setValue(inp)
  
  var inpTxt = inp.message.text
  var chatId = inp.message.chat.id
  
  var link = inpTxt.replace('https://vk.com/','').replace('vk.com/','').replace('^club','')
  //  
  // start
  if (inpTxt == '/start' || inpTxt == ' '){
    sendText(chatId, '      Google .\n'+
                    '/start -   \n'+
                    '/sort - \n'+
                    '/getvkfave -    \n'+
                    '/addinfo -  \n'+
                    '/getall -   ')
    return true
  }
  // sort
  if (inpTxt == '/sort' || inpTxt == ''){
    sort()
    sendText(chatId, '')
    return true
  }
  // getvkfave
  if (inpTxt == '/getvkfave' || inpTxt == '   '){
    getVkFave(true)
    sendText(chatId, '')
    return true
  }
  // addinfo
  if (inpTxt == '/addinfo' || inpTxt == ' '){
    if (addInfo(true)) sendText(chatId, '')
    else sendText(chatId, ' ,   ')
    return true
  }
  // getall
  if (inpTxt == '/getall' || inpTxt == '  '){
    var arr = [[],[],[],[],[],[],[],[],[],[],[' ']] // TODO 
    var repeat = ''
    var n = 0
    var j = 0
    for (var i = 1; i < data.length; i++) {
      if (j == 4){ //    
        n++
        j = 0
      }
      if (repeat != data[i][3]){
        arr[n][j] = data[i][3]
        if (data[i][3] == '') arr[n][j] = '---'
        j++
      }
      repeat = data[i][3]
    }
    var key = JSON.stringify({keyboard: 
                              arr,
                              resize_keyboard:true,
                              one_time_keyboard:true
                            });
    sendText(chatId, ' ', key)
    return true
  }
  
  // 
  if (isRus(link) != -1) {
    var name = (inpTxt == '---')?' ':inpTxt
    for (var j = 1; j < data.length; j++) {
      if (data[j][3] == inpTxt || inpTxt == '---') {
        var str = ' '+name+'\n\n'
        for (var i = 1; i < data.length; i++) {
          if ((data[j][3] == data[i][3]) || (data[i][3] == '' && inpTxt == '---')) {
            if (str.length >= 4000) { //   -    Telegram 
              sendText(chatId, str)
              str = ''
            }
            str = str + '<a href="https://vk.com/' + data[i][2] + '">' + escapeHtml(data[i][0] + data[i][1]) + '</a> \n'
          }
        }
        if (str != '') sendText(chatId, str)
        return true
      }
    }
    sendText(chatId, inpTxt + ' -   .    ')
    return true
  }
  //   
  if (getStatusUrl('https://vk.com/'+link) != 200 && getStatusUrl('https://vk.com/club'+link) != 200){
    sendText(chatId, inpTxt+' -  ')
    return true
  }
  
  var name = searchExists(link)
  if (name != false){
    sendText(chatId, ' '+name+'  ')
    return true
  }else{
    var html = UrlFetchApp.fetch('https://api.vk.com/method/groups.getById.json?group_id='+link+'&fields=city,description&access_token='+access_token+'&v=5.103')
    var json = JSON.parse(html).response[0]
    addInTableFromArray(json, link)
    sort()
    sendText(chatId, ' '+json.name+' ')
    return true
  }
}
//  
function sendText(chatId, text, key = ''){
  var payload = {
    'method': 'sendMessage',
    'chat_id': String(chatId),
    'text': text,
    'parse_mode': 'HTML',
    'reply_markup': key,
    'disable_web_page_preview': true,
    'one_time_keyboard':true
  }     
  var data = {
    "method": "post",
    "payload": payload
  }
  //    UrlFetchApp.fetch   ll(text)
  UrlFetchApp.fetch(botApi, data)
}


More details below.

Click " Publish - Deploy as a web application ."



Now we have a link of the form script.google.com/macros/s/AAA/exec , substituting the “link” parameter to it, we can add the VK group to Google Table.

User scripts in the browser


The first and main way of entering data!

Open the Google Chrome browser or Mozilla Firefox and install the Tampermonkey user script plugin.

Links for Chrome , for Mozilla .

Originally used Greasemonkey, but had to switch to Tampermonkey due to cross-browser compatibility.

Who will have questions why the user script parameters start with the GM_ prefix , and not TM_ - the idea is not mine, go to the documentation .


We insert the vkGroupToGS script by clicking on the first tab with a plus sign.

vkGroupToGS
// ==UserScript==
// @name         vkGroupToGS
// @namespace    https://vk.com/*
// @version      0.1
// @author       You
// @match        https://vk.com/*
// @grant        GM_xmlhttpRequest
// ==/UserScript==
    var url_first = 'https://script.google.com/macros/s/AAA/exec'
    var url_short = document.location.href.replace("https://vk.com/", "")
    var d = document.createElement('div')
    var head = document.querySelector('.left_menu_nav_wrap')
    d.setAttribute('id', 'send_group')
    d.style.display = 'inline-block'
    d.style.position = 'relative'
    d.style.fontSize = '50pt'
    d.style.cursor = 'pointer'
    d.innerHTML = '+'
    head.parentNode.appendChild(d)
    d.onclick = function() {
    setTimeout(function() {
        GM_xmlhttpRequest({
            method: 'GET',
            url: url_first + '?link=' + url_short,
            headers: {
                'Accept': 'application/atom+xml,application/xml,text/xml'
            },
            onload: function(x) {
                console.log(x.responseText)
                if (//.test(x.responseText)) document.querySelector('#send_group').style.color = 'red'
                else document.querySelector('#send_group').style.color = 'green'
            }
        })
    }, 0)
};


"File - Save."


As you can see, our script draws a plus sign (which was enough for imagination) in the left block left_menu_nav_wrap .


By clicking on the plus sign, GM_xmlhttpRequest from vkGroupToGS is executed in our script Code.gs using the Google Sheets link attached above (url_first).

Tampermonkey will ask us to associate these 2 scripts with rights, click either " Always allow " or " Always allow domain " for " script.google.com ". Repeat as needed for " script.googleusercontent.com ".


As soon as the request successfully flies away, the callback from GM_xmlhttpRequest will come and color our button green upon successful addition or red if the error returns.

Plus Tampermonkey - cloud-based storage of scripts in Google (previously not) in the presence of their fertility and their imagination.

Back to Google Sheets.

Before starting to parse our Code.gs, I want to share the documentation for development in VK - vk.com/dev

VK token


Obtaining an access key.
Making requests to the VKontakte API.

So far I have learned only one way to get a token - through self-written applications.

We come here .
Platform - " Standalone application ".
After creating, all we need is an application ID (now seven-digit), let it be 1234567.
But we need a token.
Very well about the token is written in Meet the VK API .

If we don’t want to read, just open the link by inserting your application ID into client_id . We are redirected to a page with the inscription:


Please do not copy the data from the address bar for third-party sites. Thus, you may lose access to your account.

And that's right, be very careful with the token!

We copy the token from the address bar after access_token .

I’ll immediately share that with this token you can query like vk.com/dev/groups.search?params%5Bq%5D=%D0%BC%D1%83%D0%B7%D1%8B%D0%BA%D0%B0¶ms % 5Btype% 5D = page & params% 5Bcountry_id% 5D = 1 & params% 5Bcity_id% 5D = 1 & params% 5Bmarket% 5D = 0 & params% 5Boffset% 5D = 0 & params% 5Bcount% 5D = 5 & params% 5Bcount% 5D = 5 & params% 5Bcount% 5D = 5 & params% 5Bcount7D

can be safely converted to links api.vk.com/method/groups.search?q=%D0%BC%D1%83%D0%B7%D1%8B%D0%BA%D0%B0&type=page&country_id=1&city_id=1&future= 0 & market = 0 & offset = 0 & count = 10 & access_token = 1111111111111 & v = 5.107 and receive information in any application.



I opened a small Pandora’s box, maybe this is not news, but in this way you can get a lot of useful things from VK, automate many processes and make a lot of convenient things.

Of course, with certain limitations , let's say no more than 999 entities in the response or 20 requests per second, etc.

Also, someone will be interested in playing with the stored files - in our application the tab " Stored Procedures ".

It has its own rules for pulling information, which is very convenient for sequential queries, so as not to make many loops in your code.

Follow the versions!

With their changes, the functionality of the methods may change. Now I am applying examples for version 5.107 .

So, we have the token, go to ...

Code.gs

doGet (e) - We accept incoming get requests from the bot, VK and just from the browser.
The input parameters are parsed, like e.parameters ['INPUT_Parameter']
A link arrives, check to see if there is a table. If not, request detailed information via groups.getById , parsim, add.

! Attention! The script is simple, so it will swallow all incoming links, so be careful even entering 123 when checking the script.

searchGenre - a smart search for the style / genre of the group, because of which I put the label “regular expressions” in the article.

Improvements are possible.

While it works only to search for the subsequent string of words through a slash or hyphen after the words "style or genre or genre ".

isRus is a simple regular rule excluding Russian letters.
sort - sorting. I set the default to 1. city, 2. name.
searchExists - simple search for existing groups through a loop.
addInfo - made a script for myself, left it here , as a bonus. It is necessary if you have a list of links, but there is no data on them.

Insert the list into the "Link" column, click " Refill empty " in the main menu and the remaining columns are replenished from VK.

Due to restrictions on the number of characters in UrlFetchApp .fetch and in the number of links placed in the address bar had to make a cycle of 1000 characters.

getVkFave- Another bonus that occurred in my head while writing an article and which I myself began to use.

The bottom line is we go into the VC application from the phone, stumble upon the group that we want to save, save it in bookmarks with the “ music grouplabel added in advance (global variable faveTag ).

How do they accumulate, go to our Google Table, call up the menu, click " Retrieve bookmarks from VK ".

The script with the token crawls into api.vk.com/method/fave.getTags , parses, adds to the table, having previously warned of removal from the VC (here, more accurately with testing, everything is deleted completely from bookmarks with this tag ).

getStatusUrl and checkActualLink- an experiment that resulted in 2 scripts.
The bottom line is to check the validity and access to links.

Because over the years of accumulation of musical groups, I was faced with the fact that public records are deleted or renamed.

Who will use my convenience, I advise some TODO , which will not only delete invalid links, but also check the presence in VK through groups.search by name.

checkActualLink - one-time checks the validity by coloring invalid in red.

  • Minus - low speed.
  • Plus - you can do it when you want.

getStatusUrl - called in the cell through a script

=getStatusUrl(CONCAT("https://vk.com/";C2))

He took the basis from here .

  • Minus - runs for all cells each time the page is opened.
  • — , , checkActualLink.

Next come the convenience for google scripts:

addInTableFromArray - Add a row to the table. Either from the cycle or one-time.
getNumSheet - Search for sheet number by name. Honestly, I didn’t find a simpler way on the Internet than this one. It is necessary due to the shift of sheet numbers when adding new ones. He took the basis from here .
escapeHtml - Delete special characters
ll - made Logger.log short for itself , reduces time, I advise.
onOpen - A familiar feature for drawing menus in Google Sheets.

So, our Code.gs catches links from VK through Tampermonkey and adds groups to Google Table.

Turn on laziness


  • We are sitting at the computer.
  • We are sitting on the phone.

From a computer, we learned how to add groups through the " plus sign " in VK and through bookmarks.
Through the phone, only through bookmarks in the VK application.

Let's look at two more options for making groups.

USI


User scripts in a mobile browser.

To some it will seem old-fashioned and unnecessary, but there will be those people who will take the pros out of it.

Searching the Internet so far found only such a plugin .
It works only in Firefox (tested only in android).

A huge plus - you can use the Tampermonkey script.

Due to the differences in layout between vk.com and m.vk.com , and also because of opening the default window in m.vk.com, I did not draw a plus sign , but made a simple script VK_event_to_list_mvk that works when the window opens (burn me in hell for setTimeout , I know).

Code.gs
// ==UserScript==
// @name         VK_event_to_list_mvk
// @namespace    https://m.vk.com/*
// @match        https://m.vk.com/*
// @grant    GM_xmlhttpRequest
// ==/UserScript==
setTimeout(function() {
    var url_first = 'https://script.google.com/macros/s/111/exec';
    var url_short = document.location.href.replace("https://m.vk.com/", "");
    GM_xmlhttpRequest({
        method: 'GET',
        url: url_first + '?link=' + url_short,
        headers: {
            'Accept': 'application/atom+xml,application/xml,text/xml'
        },
        onload: function(x) {
            alert(x.responseText);
        }
    });
}, 2000);



Telegram Bot


We create a bot according to the instructions .
We need a token, let it be 123123.

Create a Bot.gs script , insert our 123123 into the botApi variable , framing it with quotes and the api.telegram.org link .

Next, we need to link our bot with Google Sheets. You can see how to do this via WebHook in the previous article .

What will our bot do?

We decompose by the commands:

  • / start - Start the bot
  • / sort - Sort
  • / getvkfave - Retrieve bookmarks from VK
  • / addinfo - Fill in empty
  • / getall - Get all groups

We see that we have the beginning of the bot, there we will insert just a tutorial with a descriptive part at your discretion.

Three commands from the main menu just in case.

And the next bonus is / getall , about it below.

Commands for convenient calling (write a slash in the message input line) can be added to the same @BotFather via / setcommands.

Format - lines through a dash:

start -   
sort - 
getvkfave -    
addinfo -  
getall -   


Bot.gs

doPost - accepts incoming messages from the bot.

Naturally, as an inquisitive programmer, I immediately started using logs. Perhaps there are easier ways, but I just created a new sheet, called it “Logs” and I catch incoming requests in the first cell.

Therefore, after we parsed the incoming message through JSON.parse (e.postData.contents) , we immediately write to the logs.

Next, we determine which of the teams came to us.

If not a team, we check for validity, that is, does a group exist in VK.

sendText - sends messages to the bot (default parse_mode = HTML), documentation .

/ getall - we will analyze it separately.

Don't be afraid of my decision from

[[],[],[],[],[],[],[],[],[],[],[' ']]

Made to fill the lines with cities, while we restrict ourselves to 8-10 lines in 4 cities (variable j = 4).

If the music group is without a city, insert ---.

Just choose the city and catch Russian letters in Bot.gs .

We display a list of links, nothing more.


Remember that access to documentation and testing the bot through the address bar through the Hotspot Shield Free VPN Proxy is Unlimited VPN (for Google Chrome).

So we learned three ways to enter our table:

  • User scripts in the browser
  • User Scripts in a Mobile Browser (USI)
  • Telegram Bot

What you need to know and remember?


Club prefix in VK links VK

groups have 2 types of identifier storage.

  1. The name is in English with valid underscores and hyphens.
  2. ID

And they can give all the best and so on.

Check is simple.

If there is a group, let's say vk.com/4soulsband , substitute 4soulsband into group_ids in vk.com/dev/groups.getById , a list of parameters is displayed, one of which is an ID . And if we put club in front of him , voila, we get to the same page vk.com/club68130764 .

Limiting the storage of groups in VK to 5000

It seems ridiculous, but I didn’t have enough.

My pain with updating the VK page

I confess until I was able to defeat the catch of the transition between pages in VK to reset the plus sign in Tampermonkey. TODO

Using Google Sheets as a DB

  • Plus - availability
  • Minus - when more than 5000 lines are accumulated, work with the " DB " slows down . So for serious projects it is better to use normal DBMS.

Definition of a group city The group

address in VK can be here vk.com/dev/groups.getAddresses , respectively, who wants to improve the search for a city for a group can use it.

Total


We have:

  1. Google data collection table with which we learned to work.
  2. Knowledge of VK API. That is, you are already aware that we can not be limited to musical groups.
  3. Telegram Bot with a database that does not need to be deployed and configured.

My goal is not to boast of the next created toy, my goal is to show newcomers (and not only) a huge layer of open APIs that can help make life to one degree or another, move from the iron earth to the "clouds", so to speak.

This functionality is ready, as is open for improvements.

Good luck in the development and comprehension of new knowledge.

Documentation


Google Apps Script - Spreadsheet Service
Telegram Bot API
VK API
Learn to write userscripts
Userscripts. Deepening
Site Monitoring Using Google Docs
Pampering. We write Telegram bot on Google script
[Examples, Google Apps Script] Development of add-ons / scripts for Google Sheets (spreadsheets)
How to use google spreadsheets to check for broken links

All Articles