mathieu.poirier/tooling.git - [no description]

#
# SPDX-License-Identifier: GPL-2.0
# (c) 2020,2022, Mathieu Poirier <mathieu.poirier@linaro.org>
#

1 ## Install GNU's parallel utility:

mpoirier@xps15:~/work$ sudo apt-get install parallel

2 ## Install git:

mpoirier@xps15:~/work$ sudo apt-get git

3 ## Add clone tools and add them to system PATH:

mpoirier@xps15:~/work$ git clone https://git.linaro.org/people/mathieu.poirier/tooling.git
mpoirier@xps15:~/work$ cd tooling
mpoirier@xps15:~/work$ ls
mpoirier@xps15:~/work/kwg-523/test$ ls tooling/
get-mainline-commit  get-mainline-feature  negative-set  patchnet-set  patchnet-write  positive-set  README.txt
mpoirier@xps15:~/work/tooling$ export PATH=$PATH:/home/mpoirier/work/tooling
mpoirier@xps15:~/work/tooling$ chmod 755 *

4 ## Download the PatchNetTool model:

mpoirier@xps15:~/work$ mkdir patchnet
mpoirier@xps15:~/work$ cd patchnet
mpoirier@xps15:~/work/patchnet$ git init .
mpoirier@xps15:~/work/patchnet$ git remote add hvdthong/PatchNetTool.git https://github.com/hvdthong/PatchNetTool.git

mpoirier@xps15:~/work/patchnet$ git fetch hvdthong/PatchNetTool.git
mpoirier@xps15:~/work/patchnet$ git checkout -b PatchNetTool.git/master hvdthong/PatchNetTool.git/master

5 ## Download the PatchNet_updated model:

mpoirier@xps15:~/work$ mkdir patchnet_updated
mpoirier@xps15:~/work$ cd patchnet_updated
mpoirier@xps15:~/work/patchnet_updated$ git init .
mpoirier@xps15:~/work/patchnet_updated$ git remote add Xin-Zhou-smu/PatchNet_updated.git https://github.com/Xin-Zhou-smu/PatchNet_updated.git

mpoirier@xps15:~/work/patchnet_updated$ git fetch Xin-Zhou-smu/PatchNet_updated.git
mpoirier@xps15:~/work/patchnet_updated$ git checkout -b PatchNet_updated.git/patch-1 Xin-Zhou-smu/PatchNet_updated.git/patch-1

6 ## Create kernel directory and download source:

mpoirier@xps15:~/work$ mkdir kernel
mpoirier@xps15:~/work$ cd kernel
mpoirier@xps15:~/work/kernel$ git init .
mpoirier@xps15:~/work/kernel$ git remote add torvalds/linux.git git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
mpoirier@xps15:~/work/kernel$ git remote
torvalds/linux.git
mpoirier@xps15:~/work/kernel$ git fetch torvalds/linux.git

7 ## Checkout the latest version:

mpoirier@xps15:~/work/kernel$ git branch -r 
torvalds/linux.git/master
mpoirier@xps15:~/work/kernel$ git checkout -b torvalds/linux.git/master-v5.8 v5.8
Switched to a new branch 'torvalds/linux.git/master-v5.8'

8 ## Get a handle on the stable trees:

IMPORTANT: For the time being the name used for the stable trees must be the
same as what follows.  See script "positive-set" for details.

mpoirier@xps15:~/work/kernel$ git remote add stable/linux-stable.git git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
mpoirier@xps15:~/work/kernel$ git fetch stable/linux-stable.git
mpoirier@xps15:~/work/kernel$ git branch -r | grep linux-stable
  stable/linux-stable.git/linux-2.6.11.y
  stable/linux-stable.git/linux-2.6.12.y
  stable/linux-stable.git/linux-2.6.13.y 
...
...
...
  stable/linux-stable.git/linux-5.5.y
  stable/linux-stable.git/linux-5.6.y
  stable/linux-stable.git/linux-5.7.y
  stable/linux-stable.git/master

9 ## Generate the positive training set by gathering all the patches that have
   been backported to stable kernels:

mpoirier@xps15:~/work/kernel$ positive-set positive.txt
(depending on the machine, this can take hours to complete)

mpoirier@xps15:~/work/kernel$ wc -l positive.txt
44397 positive.txt

10 ## Generate the negative training set using the output, i.e file positive.txt,
   from the above command:

mpoirier@xps15:~/work/kernel$ negative-set positive.txt negative.txt v5.0..v5.8 
(depending on the machine, this can take hours to complete)

mpoirier@xps15:~/work/kernel$ wc -l negative.txt
82121 negative.txt

11 ## Generate the input file to train PatchNetTool by using the output of the above
   two commands:

mpoirier@xps15:~/work/kernel$ patchnet-set -p positive.txt -n negative.txt train.txt
mpoirier@xps15:~/work/kernel$ wc -l train.txt
126518 training-set.txt

12 ## Generate the input file to test new patches:

mpoirier@xps15:~/work/kernel$ testing-set testing.txt v5.8..v5.9

13 ## Use the training and test sets throught the preprocessing phase:

mpoirier@xps15:~/work$ cp kernel/train.txt patchnet/preprocessing/
mpoirier@xps15:~/work$ cp kernel/test.txt patchnet/preprocessing/
mpoirier@xps15:~/work$ cd patchnet
mpoirier@xps15:~/work/patchnet/preprocesing$ ./getinfo -j 36 --commit-list train.txt --git /home/linaro/kernel -o train
(Depeding on the machine and the size of the training set, this can take hours)
mpoirier@xps15:~/work/patchnet/preprocessing$ ./getinfo -j 36 --commit-list testing.txt --git /home/linaro/kernel -o testing
(Depeding on the machine and the size of the training set, this can take hours)

14 ## Generate .pkl data from .out files:

mpoirier@xps15:~/work/patchnet/preprocessing$ cp train.out testing.out ../../patchnet_updated/
mpoirier@xps15:~/work/patchnet$ cd ../../patchnet_updated/

mpoirier@xps15:~/work/patchnet_updated$ python text2dict.py -text_path train.out -dict_path train.pkl

mpoirier@xps15:~/work/patchnet_updated$ python text2dict.py -text_path  'testing.out' -dict_path 'test.pkl'

mpoirier@xps15:~/work/patchnet_updated$ python generate_dict.py -text_path1 train.out -text_path2 testing.out -dict_path dict.pkl

15 ## Train the model:

mpoirier@xps15:~/work/patchnet_updated$ python main.py -train -train_data train.pkl -dictionary_data dict.pkl