{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Table of Content","type":"text","marks":[{"type":"strong"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Objective ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Hardware Requirements ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"IAM user and Role creation for S3 connectivity ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"IAM user and Role creation for AZURE Storage Blob Container connectivity","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Local SDC Installation","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"cQube Deployment Process ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Appendix ","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"AWS - Network Architecture ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Microservice Details ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"cQube Deployment Procedure ","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Objective","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"This document mainly focuses on the hardware requirements/ AWS network architecture for installing cQube and the deployment process step-by-step.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Hardware Requirements","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" Create an AWS EC2 or AZURE VM or in LOCAL SDC or Oracle VM instance with below configurations to install all the cQube micro services.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ubuntu 22.04 (supported) ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"16 GB of System RAM (minimum requirement)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"4 core CPU for AWZ, AZURE, LOCAL / 2 OCPU for ORACLE (minimum requirement)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"250GB HDD","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Security Group Configuration:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Port 80 inbound from 0.0.0.0/0","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Port 443 inbound from 0.0.0.0/0","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"5432 inbound to the particular ip which needs access","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Kong Configurations: ( already configured using one step deployment )","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"3000 will get routed to /ingestion","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"3001 will get routed to /spec","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"3003 will get routed to /generator","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Domain Name:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a domain name","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Configure cname of AWS ec2 or AZURE VM or LOCAL sdc or ORACLE VM instance to the domain name","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a SSL certificate for the domain name.","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"IAM user and Role creation for S3 connectivity (AWS)","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"An AWS Identity and Access Management (IAM) user is an entity that is created in AWS to represent the person or application that uses it to interact with AWS. A user in AWS contains a name and credentials. An IAM user with administrator permissions is different from the AWS account root user. One has to create an IAM user with a supported role to provide the connectivity between EC2 and S3. The role should have list, read and write permissions.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"IAM user and Role creation for AZURE Storage Blob Container connectivity","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"An AZURE Identity and Access Management (IAM) user is an entity that is created in AZURE to represent the person or application that uses it to interact with AZURE. A user in Azure contains a name and credentials. An IAM user with administrator permissions is different from the AWS account root user. One has to create an IAM user with a supported role to provide the connectivity between azure VM and blob container. The role should have list, read and write permissions.","type":"text"}]},{"type":"paragraph","content":[{"text":"IAM User:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a IAM user","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Assign IAM policy to the user","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Download the Default Connection String and Azure account name and account name and container name","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"IAM Policy:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a IAM policy from AZURE IAM","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Provide access to list, read and write the objects to azure Blob container","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"For Local SDC installation","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"If a user opting for local installation, Enter storage_type as local. Then automatically install.sh will install and configure the minio application in localhost and it will set default username and password and it will create a minio bucket as well.","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"cQube Deployment Process FOR AWS","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Step - 1:","type":"text","marks":[{"type":"strong"}]},{"text":" Connect to the cqube AWS ec2 instance","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For linux and macOS:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Download the .pem file which is generated while creating the EC2 instance","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Open the terminal and navigate to the folder where .pem file has been downloaded","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Then give the read permission to the .pem file using following command","type":"text"}]}]}]}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":" sudo chmod 400 ","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Use the following command to connect to the instance","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"ssh -i @","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For windows:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Download the .pem file which is generated while creating the EC2 instance","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Use puttygen to connect to the instance.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Refer following link to use puttyGen for connecting.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/putty.html"}},{"text":" ","type":"text"}]}]}]}]}]}]}]},{"type":"paragraph","content":[{"text":"Upgradation process for AZURE","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Step - 1:","type":"text","marks":[{"type":"strong"}]},{"text":" Connect to the cqube Azure VM instance","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For linux and macOS:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Download the .pem file which is generated while creating the azure VM instance","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Open the terminal and navigate to the folder where .pem file has been downloaded","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Then give the read permission to the .pem file using following command","type":"text"}]}]}]}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":" sudo chmod 400 ","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Use the following command to connect to the instance","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"ssh -i @","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For windows:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Download the .pem file which is generated while creating the azure VM instance","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Use puttygen to connect to the instance.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Refer following link to use puttyGen for connecting.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/putty.html"}},{"text":" ","type":"text"}]}]}]}]}]}]}]},{"type":"paragraph","content":[{"text":"Upgradation process for LOCAL","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Prerequisites to install cQube on local machine","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph"},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ubuntu 22.04 (supported)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" 16 GB of System RAM (minimum requirement)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" 4 core CPU (minimum requirement)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" Domain name (with SSL)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" 250 GB Storage","type":"text"}]}]}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Step - 2:","type":"text","marks":[{"type":"strong"}]},{"text":" Clone the cqube-devops repository using following command","type":"text"}]},{"type":"paragraph","content":[{"text":"git clone ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/Sunbird-cQube/cqube-devops.git"}},{"text":" ","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":959,"id":"7bc9bc1f-a0ee-4c26-94aa-57b76a25b3b1","collection":"contentId-3280666668","type":"file","height":162}}]},{"type":"paragraph","content":[{"text":"Step - 3:","type":"text","marks":[{"type":"strong"}]},{"text":" Navigate to the directory where cqube is cloned or downloaded and checkout to the desired release branch","type":"text"}]},{"type":"paragraph","content":[{"text":"cd cqube-devops/ && git checkout ","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":959,"id":"4715e66c-f4dc-47c8-ae35-7903a5578925","collection":"contentId-3280666668","type":"file","height":76}}]},{"type":"paragraph","content":[{"text":"Step - 4:","type":"text","marks":[{"type":"strong"}]},{"text":" Give the following permissions to the install.sh file","type":"text"}]},{"type":"paragraph","content":[{"text":"sudo chmod u+x install.sh","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Step - 5:","type":"text","marks":[{"type":"strong"}]},{"text":" Install cqube with non root user with sudo privileges","type":"text"}]},{"type":"paragraph","content":[{"text":"sudo ./install.sh","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":611,"id":"45532513-abf7-4faf-8073-c18733dcb383","collection":"contentId-3280666668","type":"file","height":53}}]},{"type":"paragraph","content":[{"text":"Install.sh file contains a shell script where it will run shell scripts and ansible-playbook to setup the cqube","type":"text"}]},{"type":"paragraph","content":[{"text":"Step - 6:","type":"text","marks":[{"type":"strong"}]},{"text":" User Input Variables - These are the variables which need to be entered by the user by following the Hint provided","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"state_name ( Enter the required state code by referring to the state list provided )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"api_end_point ( Enter the url in which cqube to be configured )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage_type (Enter the storage_type as aws or local or azure. If User opting for aws You will be prompted with the following AWS s3 credentials to enter and it will create the s3 bucket. If s3_bucket already exists it will prompt you to enter a unique s3 bucket name. And it will be generated in the config.yml.)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3_access_key","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3_secret_key","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3 bucket name","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1323,"id":"e4e827a5-4dc1-46e0-bae4-a3769db8449d","collection":"contentId-3280666668","type":"file","height":353}}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage_type (Enter the storage_type as local, minio will install and configure the minio username and minio password and create the minio bucket And it will be generated in the config.yml.)","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1323,"id":"8cde66cc-8205-4b0b-b40c-3e3ce85b6c20","collection":"contentId-3280666668","type":"file","height":111}}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":960,"id":"76517e6f-76c5-461e-af26-0912958a8796","collection":"contentId-3280666668","type":"file","height":112}}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage_type (Enter the storage_type as Azure, If User opting for Azure you will be prompted with the following Azure credentials to enter and it will create an azure container , if azure container exists it will prompt to enter a unique azure container name. And it will be generated in the config.yml.)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure_connection_string","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure_account_name","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure_account_key","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1323,"id":"e9a22142-f668-4ab5-99ff-b659ba96226c","collection":"contentId-3280666668","type":"file","height":221}}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Step - 7:","type":"text","marks":[{"type":"strong"}]},{"text":" Optional_variables- Database credentials contain default values. If the user wishes to enter their own credentials then the user should opt for ","type":"text"},{"text":"yes ","type":"text","marks":[{"type":"strong"}]},{"text":"to enter their credentials otherwise can opt for ","type":"text"},{"text":"no","type":"text","marks":[{"type":"strong"}]},{"text":" when the question pops up","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_user_name ( Enter the postgres database username ) ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_name ( Enter the postgres database name )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_password ( Enter the postgres password )","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1320,"id":"150aa5d5-b6d9-4b4a-92c8-24bf911534b7","collection":"contentId-3280666668","type":"file","height":131}}]},{"type":"paragraph","content":[{"text":"Step - 8:","type":"text","marks":[{"type":"strong"}]},{"text":" Optional_variables- Read Only Database credentials contain default values. If the user wishes to enter their own credentials then the user should opt for ","type":"text"},{"text":"yes ","type":"text","marks":[{"type":"strong"}]},{"text":"to enter their credentials otherwise can opt for ","type":"text"},{"text":"no","type":"text","marks":[{"type":"strong"}]},{"text":" when the question pops up","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"read_only_db_user ( Enter the read only postgres database username ) ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"read_only_db_password ( Enter the read onlypostgres password )","type":"text"}]}]}]},{"type":"paragraph"},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1327,"id":"acd2bb04-95c8-4ba7-b22c-2a237b1361ef","collection":"contentId-3280666668","type":"file","height":115}}]},{"type":"paragraph","content":[{"text":"Step - 9:","type":"text","marks":[{"type":"strong"}]},{"text":" Keycloak_variables- Keycloak credentials with Keycloak admin user name and keycloak admin password.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Keyclaok_adm_name (Enter the keycloak admin name eg: admin)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Keyclaok_adm_password (enter the keycloak admin password eg: Admin@123)","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1392,"id":"fc673e29-a33c-41a5-b5fd-59e74214843d","collection":"contentId-3280666668","type":"file","height":132}}]},{"type":"paragraph","content":[{"text":"Step - 10:","type":"text","marks":[{"type":"strong"}]},{"text":" Once the config file is generated, A preview of the config file is displayed followed by a question where the user gets an option to re enter the configuration values on choosing ","type":"text"},{"text":"yes. I","type":"text","marks":[{"type":"strong"}]},{"text":"f option ","type":"text"},{"text":"no ","type":"text","marks":[{"type":"strong"}]},{"text":"is selected then the install.sh moves to the next section.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1127,"id":"38abf3da-5af9-4f85-95e0-42a7c95de214","collection":"contentId-3280666668","type":"file","height":420}}]},{"type":"paragraph","content":[{"text":"Step - 11:","type":"text","marks":[{"type":"strong"}]},{"text":" A preview of the program_selector.yml file is displayed followed by a question where the user gets an option to enable or disable the programs on choosing ","type":"text"},{"text":"yes. I","type":"text","marks":[{"type":"strong"}]},{"text":"f option ","type":"text"},{"text":"no ","type":"text","marks":[{"type":"strong"}]},{"text":"is selected then the ","type":"text"},{"text":"install","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"669559dc-3579-46bd-8bee-016af79b8d73"}}]},{"text":".sh moves to the next section.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":959,"id":"3cd7216f-50db-4ceb-b6a7-7da0e2bc6699","collection":"contentId-3280666668","type":"file","height":258}}]},{"type":"paragraph","content":[{"text":"Step - 12:","type":"text","marks":[{"type":"strong"}]},{"text":" Once the installation is completed, You will be prompted with the following messages and required reference urls.","type":"text"}]},{"type":"paragraph","content":[{"text":"cQube Installed Successfully!!","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1398,"id":"4e6ce2b5-6ee3-49a4-9a80-cccb2ff98449","collection":"contentId-3280666668","type":"file","height":276}}]},{"type":"paragraph","content":[{"text":"cQube ingestion api can be accessible using ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Ingestion Usage Documentation :","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://project-sunbird.atlassian.net/l/cp/PPn7AfAW"}}]},{"type":"inlineCard","attrs":{"url":"https://project-sunbird.atlassian.net/l/cp/PPn7AfAW"}},{"text":" ","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Schema Documentation :","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://project-sunbird.atlassian.net/l/cp/xpVi7HbS"}}]},{"type":"inlineCard","attrs":{"url":"https://project-sunbird.atlassian.net/l/cp/xpVi7HbS"}},{"text":" ","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"UI Usage Documentation :","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://project-sunbird.atlassian.net/l/cp/mv7JvLXd"}},{"text":" ","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Appendix","type":"text","marks":[{"type":"strong"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"AWS - Network Architecture","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"The following steps define the cQube setup and workflow completion processes in AWS. cQube mainly comprises the areas mentioned below:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"EC2 Server","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"IAM user and Role creation for S3 connectivity.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The cQube network setup process is described in the block diagram below:","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":611,"id":"f1760e5c-591c-416d-b271-c94908af4da2","collection":"contentId-3280666668","type":"file","height":511}}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Microservices Details","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" Following are the details of the microservices which get installed in the cqube server.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ingestion-ms:","type":"text","marks":[{"type":"strong"}]},{"text":" The ingestion-ms is used to upload the data of the events, datasets, dimensions, transformers and pipeline. All these apis will be to ingesting the data into the cQube.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Spec-ms:","type":"text","marks":[{"type":"strong"}]},{"text":" The spec-ms is used to import schema of the events, datasets, dimensions, transformers and pipeline. All these specs will be defined by the cQube platform prior to ingesting the data into the cQube. These specifications are derived by considering the KPIs as the Indicator.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Generator-ms: ","type":"text","marks":[{"type":"strong"}]},{"text":"The generator-ms is used to create the specs & transformers for the derived datasets. Performed aggregation logics, updating data to datasets based on transformation. Status update of file processing","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Nifi-ms: ","type":"text","marks":[{"type":"strong"}]},{"text":"Apache NiFi is used as a real-time integrated data logistics and simple event processing platform","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Postgres-ms: ","type":"text","marks":[{"type":"strong"}]},{"text":"Postgres microservice contains the schema and tables","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Nginx-ms: ","type":"text","marks":[{"type":"strong"}]},{"text":"It is commonly used as a reverse proxy and load balancer to manage incoming traffic and distribute it to slower upstream servers","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kong-ms:","type":"text","marks":[{"type":"strong"}]},{"text":" It is a lightweight API Gateway that secures, manages, and extends APIs and microservices.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Dashboard-ms","type":"text","marks":[{"type":"strong"}]},{"text":": It consists of an angular app, it is used to visualize the datasets present in postgres-ms in the form of charts. On run time it requests spec-ms to fetch data from postgres-ms and load it into the client side(Browser)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Query_builder-ms: ","type":"text","marks":[{"type":"strong"}]},{"text":"It consists of backend API, which consists of JWT,METRICS,QUERY apis ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"JWT - it will generate a jwt token to restrict the other apis.","type":"text"}]},{"type":"paragraph","content":[{"text":"METRICS - it consists of menus for the navigation bar and dashboard cards.","type":"text"}]},{"type":"paragraph","content":[{"text":"QUERY - this api used for executing the SQL queries which integrated with Dashboard-ms.","type":"text"}]},{"type":"paragraph","content":[{"text":"LASTMODIFIED - this api will use for the last modified data in the s3,azure and minio","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"cQube Deployment Procedure","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":" Install.sh file contains a shell script where it will run following shell scripts and ansible-playbook to setup the cqube","type":"text"}]},{"type":"paragraph","content":[{"text":"Basic_requirements.sh:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"This script basically updates and upgrades the software packages in the server and installs the basic softwares such as ","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Python3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Pip3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ansible","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Docker","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Docker compose","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Config_file_generator.sh:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"This script is used to generate a configuration file which contains some constant values and few required variables should be entered by the user. Following are the variables which get added in the config file.","type":"text"}]},{"type":"paragraph","content":[{"text":"Note: ","type":"text","marks":[{"type":"strong"}]},{"text":"Users should follow the Hints provided in the description and should enter the variables accordingly. If the entered value is wrong then an error message gets displayed and the user should modify the variable value accordingly.Constant Variables: These variables are auto generated","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"System_user_name","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"base_dir","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Private_ip","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"aws_default_region","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"User Input Variables-These are variables which need to be entered by the user by following the Hint provided","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"state_name ( Enter the required state code by referring to the state list provided )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"api_end_point ( Enter the url in which cqube to be configured )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage_type (Enter the storage type as aws, azure or local","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Storage_type is aws enter the below variables","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3_access_key","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3_secret_key","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"s3 bucket name ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Storage_type is azure enter the below variables","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure connection string","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure account name","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Azure account key","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"azure container name","type":"text"}]}]}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Storage_type is a local system that will automatically create the minio username and minio password and minio bucket.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"minio username","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"minio password","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"minio bucket","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Minino can be accessed by Dashboard with ","type":"text","marks":[{"type":"strong"}]},{"text":"http://localhost","type":"text","marks":[{"type":"underline"},{"type":"strong"},{"type":"link","attrs":{"href":"http://localhost"}}]},{"text":":9001 end point, here you can see the minio bucket which is created by default and the username is minio admin and password is minio admin","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Optional_variables - Database credentials contain default values. If the user wishes to enter their own credentials then the user should opt for ","type":"text"},{"text":"yes ","type":"text","marks":[{"type":"strong"}]},{"text":"to enter their credentials otherwise can opt for ","type":"text"},{"text":"no","type":"text","marks":[{"type":"strong"}]},{"text":" when the question pops up","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_user_name ( Enter the postgres database username ) ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_name ( Enter the postgres database name )","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"db_password ( Enter the postgres password )","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Optional_variables- Read Only Database credentials contain default values. If the user wishes to enter their own credentials then the user should opt for ","type":"text"},{"text":"yes ","type":"text","marks":[{"type":"strong"}]},{"text":"to enter their credentials otherwise can opt for ","type":"text"},{"text":"no","type":"text","marks":[{"type":"strong"}]},{"text":" when the question pops up","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"read_only_db_user ( Enter the read only postgres database username ) ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"read_only_db_password ( Enter the read only postgres password )","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Once the config file is generated, A preview of the config file is displayed followed by a question where the user gets an option to re enter the configuration values on choosing ","type":"text"},{"text":"yes. I","type":"text","marks":[{"type":"strong"}]},{"text":"f option ","type":"text"},{"text":"no ","type":"text","marks":[{"type":"strong"}]},{"text":"is selected then the install.sh moves to the next section.","type":"text"}]},{"type":"paragraph","content":[{"text":"Repository_clone.sh:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"This script clones the following repositories in the microservices directory and checkout to the required release branch","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"git clone ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://github.com/Sunbird-cQube/spec-ms.git"}},{"text":" ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"git clone ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://github.com/Sunbird-cQube/ingestion-ms.git"}},{"text":" ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"git clone ","type":"text"},{"text":"https://github.com/Sunbird-cQube/generator-ms.git","type":"text","marks":[{"type":"underline"},{"type":"link","attrs":{"href":"https://github.com/Sunbird-cQube/ingestion-ms.git"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Git clone ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://github.com/Sunbird-cQube/dashboard-ms.git"}},{"text":" ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"git clone ","type":"text"},{"text":"https://github.com/Sunbird-cQube/query-builder.git","type":"text","marks":[{"type":"underline"},{"type":"link","attrs":{"href":"https://github.com/Sunbird-cQube/dashboard-ms.git"}}]},{"text":" ","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1318,"id":"bd952092-4328-4584-be10-efc64ccfa54e","collection":"contentId-3280666668","type":"file","height":240}}]},{"type":"paragraph","content":[{"text":" Note: If the repository is already cloned then the script will pull the updated code.","type":"text"}]},{"type":"paragraph","content":[{"text":"Ansible-playbook:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Install.yml","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"An install.yml ansible playbook gets triggered where it triggers the required roles to build the following microservices images.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ingestion-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Spec-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Generator-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Postgres-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Nifi-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Dashboard-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Query_builder-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kong-ms","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Nginx-ms","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1323,"id":"7e8338a0-500c-49dc-83d0-9134bc7aa10d","collection":"contentId-3280666668","type":"file","height":655}}]},{"type":"paragraph","content":[{"text":"compose.yml:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":" A docker compose ansible script gets triggered where it will up all the containers to running state.","type":"text"}]},{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" The following commands can be used from the Ansible directory to down the containers and to start the containers, respectively.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"docker-compose -f docker-compose.yml down","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"docker-compose -f docker-compose.yml up -d","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":" Once the installation is completed, You will be prompted with the following messages and required reference urls.","type":"text"}]},{"type":"paragraph","content":[{"text":"cQube Installed Successfully!!","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1398,"id":"4e6ce2b5-6ee3-49a4-9a80-cccb2ff98449","collection":"contentId-3280666668","type":"file","height":276}}]},{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"We can check the containers running status by using following command","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"sudo docker ps","type":"text"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1325,"id":"35ade80e-5c32-43e3-a310-823fc474fbb0","collection":"contentId-3280666668","type":"file","height":417}}]},{"type":"paragraph"},{"type":"paragraph"},{"type":"paragraph"}],"version":1}

Browser not supported