[{"data":1,"prerenderedAt":4802},["ShallowReactive",2],{"/en-us/blog/tags/tutorial/":3,"navigation-en-us":19,"banner-en-us":449,"footer-en-us":466,"tutorial-tag-page-en-us":676},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"content":8,"config":10,"_id":12,"_type":13,"title":14,"_source":15,"_file":16,"_stem":17,"_extension":18},"/en-us/blog/tags/tutorial","tags",false,"",{"tag":9,"tagSlug":9},"tutorial",{"template":11},"BlogTag","content:en-us:blog:tags:tutorial.yml","yaml","Tutorial","content","en-us/blog/tags/tutorial.yml","en-us/blog/tags/tutorial","yml",{"_path":20,"_dir":21,"_draft":6,"_partial":6,"_locale":7,"data":22,"_id":445,"_type":13,"title":446,"_source":15,"_file":447,"_stem":448,"_extension":18},"/shared/en-us/main-navigation","en-us",{"logo":23,"freeTrial":28,"sales":33,"login":38,"items":43,"search":376,"minimal":407,"duo":426,"pricingDeployment":435},{"config":24},{"href":25,"dataGaName":26,"dataGaLocation":27},"/","gitlab logo","header",{"text":29,"config":30},"Get free trial",{"href":31,"dataGaName":32,"dataGaLocation":27},"https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com&glm_content=default-saas-trial/","free trial",{"text":34,"config":35},"Talk to sales",{"href":36,"dataGaName":37,"dataGaLocation":27},"/sales/","sales",{"text":39,"config":40},"Sign in",{"href":41,"dataGaName":42,"dataGaLocation":27},"https://gitlab.com/users/sign_in/","sign in",[44,88,186,191,297,357],{"text":45,"config":46,"cards":48,"footer":71},"Platform",{"dataNavLevelOne":47},"platform",[49,55,63],{"title":45,"description":50,"link":51},"The most comprehensive AI-powered DevSecOps Platform",{"text":52,"config":53},"Explore our Platform",{"href":54,"dataGaName":47,"dataGaLocation":27},"/platform/",{"title":56,"description":57,"link":58},"GitLab Duo (AI)","Build software faster with AI at every stage of development",{"text":59,"config":60},"Meet GitLab Duo",{"href":61,"dataGaName":62,"dataGaLocation":27},"/gitlab-duo/","gitlab duo ai",{"title":64,"description":65,"link":66},"Why GitLab","10 reasons why Enterprises choose GitLab",{"text":67,"config":68},"Learn more",{"href":69,"dataGaName":70,"dataGaLocation":27},"/why-gitlab/","why gitlab",{"title":72,"items":73},"Get started with",[74,79,84],{"text":75,"config":76},"Platform Engineering",{"href":77,"dataGaName":78,"dataGaLocation":27},"/solutions/platform-engineering/","platform engineering",{"text":80,"config":81},"Developer Experience",{"href":82,"dataGaName":83,"dataGaLocation":27},"/developer-experience/","Developer experience",{"text":85,"config":86},"MLOps",{"href":87,"dataGaName":85,"dataGaLocation":27},"/topics/devops/the-role-of-ai-in-devops/",{"text":89,"left":90,"config":91,"link":93,"lists":97,"footer":168},"Product",true,{"dataNavLevelOne":92},"solutions",{"text":94,"config":95},"View all Solutions",{"href":96,"dataGaName":92,"dataGaLocation":27},"/solutions/",[98,123,147],{"title":99,"description":100,"link":101,"items":106},"Automation","CI/CD and automation to accelerate deployment",{"config":102},{"icon":103,"href":104,"dataGaName":105,"dataGaLocation":27},"AutomatedCodeAlt","/solutions/delivery-automation/","automated software delivery",[107,111,115,119],{"text":108,"config":109},"CI/CD",{"href":110,"dataGaLocation":27,"dataGaName":108},"/solutions/continuous-integration/",{"text":112,"config":113},"AI-Assisted Development",{"href":61,"dataGaLocation":27,"dataGaName":114},"AI assisted development",{"text":116,"config":117},"Source Code Management",{"href":118,"dataGaLocation":27,"dataGaName":116},"/solutions/source-code-management/",{"text":120,"config":121},"Automated Software Delivery",{"href":104,"dataGaLocation":27,"dataGaName":122},"Automated software delivery",{"title":124,"description":125,"link":126,"items":131},"Security","Deliver code faster without compromising security",{"config":127},{"href":128,"dataGaName":129,"dataGaLocation":27,"icon":130},"/solutions/security-compliance/","security and compliance","ShieldCheckLight",[132,137,142],{"text":133,"config":134},"Application Security Testing",{"href":135,"dataGaName":136,"dataGaLocation":27},"/solutions/application-security-testing/","Application security testing",{"text":138,"config":139},"Software Supply Chain Security",{"href":140,"dataGaLocation":27,"dataGaName":141},"/solutions/supply-chain/","Software supply chain security",{"text":143,"config":144},"Software Compliance",{"href":145,"dataGaName":146,"dataGaLocation":27},"/solutions/software-compliance/","software compliance",{"title":148,"link":149,"items":154},"Measurement",{"config":150},{"icon":151,"href":152,"dataGaName":153,"dataGaLocation":27},"DigitalTransformation","/solutions/visibility-measurement/","visibility and measurement",[155,159,163],{"text":156,"config":157},"Visibility & Measurement",{"href":152,"dataGaLocation":27,"dataGaName":158},"Visibility and Measurement",{"text":160,"config":161},"Value Stream Management",{"href":162,"dataGaLocation":27,"dataGaName":160},"/solutions/value-stream-management/",{"text":164,"config":165},"Analytics & Insights",{"href":166,"dataGaLocation":27,"dataGaName":167},"/solutions/analytics-and-insights/","Analytics and insights",{"title":169,"items":170},"GitLab for",[171,176,181],{"text":172,"config":173},"Enterprise",{"href":174,"dataGaLocation":27,"dataGaName":175},"/enterprise/","enterprise",{"text":177,"config":178},"Small Business",{"href":179,"dataGaLocation":27,"dataGaName":180},"/small-business/","small business",{"text":182,"config":183},"Public Sector",{"href":184,"dataGaLocation":27,"dataGaName":185},"/solutions/public-sector/","public sector",{"text":187,"config":188},"Pricing",{"href":189,"dataGaName":190,"dataGaLocation":27,"dataNavLevelOne":190},"/pricing/","pricing",{"text":192,"config":193,"link":195,"lists":199,"feature":284},"Resources",{"dataNavLevelOne":194},"resources",{"text":196,"config":197},"View all resources",{"href":198,"dataGaName":194,"dataGaLocation":27},"/resources/",[200,233,256],{"title":201,"items":202},"Getting started",[203,208,213,218,223,228],{"text":204,"config":205},"Install",{"href":206,"dataGaName":207,"dataGaLocation":27},"/install/","install",{"text":209,"config":210},"Quick start guides",{"href":211,"dataGaName":212,"dataGaLocation":27},"/get-started/","quick setup checklists",{"text":214,"config":215},"Learn",{"href":216,"dataGaLocation":27,"dataGaName":217},"https://university.gitlab.com/","learn",{"text":219,"config":220},"Product documentation",{"href":221,"dataGaName":222,"dataGaLocation":27},"https://docs.gitlab.com/","product documentation",{"text":224,"config":225},"Best practice videos",{"href":226,"dataGaName":227,"dataGaLocation":27},"/getting-started-videos/","best practice videos",{"text":229,"config":230},"Integrations",{"href":231,"dataGaName":232,"dataGaLocation":27},"/integrations/","integrations",{"title":234,"items":235},"Discover",[236,241,246,251],{"text":237,"config":238},"Customer success stories",{"href":239,"dataGaName":240,"dataGaLocation":27},"/customers/","customer success stories",{"text":242,"config":243},"Blog",{"href":244,"dataGaName":245,"dataGaLocation":27},"/blog/","blog",{"text":247,"config":248},"Remote",{"href":249,"dataGaName":250,"dataGaLocation":27},"https://handbook.gitlab.com/handbook/company/culture/all-remote/","remote",{"text":252,"config":253},"TeamOps",{"href":254,"dataGaName":255,"dataGaLocation":27},"/teamops/","teamops",{"title":257,"items":258},"Connect",[259,264,269,274,279],{"text":260,"config":261},"GitLab Services",{"href":262,"dataGaName":263,"dataGaLocation":27},"/services/","services",{"text":265,"config":266},"Community",{"href":267,"dataGaName":268,"dataGaLocation":27},"/community/","community",{"text":270,"config":271},"Forum",{"href":272,"dataGaName":273,"dataGaLocation":27},"https://forum.gitlab.com/","forum",{"text":275,"config":276},"Events",{"href":277,"dataGaName":278,"dataGaLocation":27},"/events/","events",{"text":280,"config":281},"Partners",{"href":282,"dataGaName":283,"dataGaLocation":27},"/partners/","partners",{"backgroundColor":285,"textColor":286,"text":287,"image":288,"link":292},"#2f2a6b","#fff","Insights for the future of software development",{"altText":289,"config":290},"the source promo card",{"src":291},"https://res.cloudinary.com/about-gitlab-com/image/upload/v1758208064/dzl0dbift9xdizyelkk4.svg",{"text":293,"config":294},"Read the latest",{"href":295,"dataGaName":296,"dataGaLocation":27},"/the-source/","the source",{"text":298,"config":299,"lists":301},"Company",{"dataNavLevelOne":300},"company",[302],{"items":303},[304,309,315,317,322,327,332,337,342,347,352],{"text":305,"config":306},"About",{"href":307,"dataGaName":308,"dataGaLocation":27},"/company/","about",{"text":310,"config":311,"footerGa":314},"Jobs",{"href":312,"dataGaName":313,"dataGaLocation":27},"/jobs/","jobs",{"dataGaName":313},{"text":275,"config":316},{"href":277,"dataGaName":278,"dataGaLocation":27},{"text":318,"config":319},"Leadership",{"href":320,"dataGaName":321,"dataGaLocation":27},"/company/team/e-group/","leadership",{"text":323,"config":324},"Team",{"href":325,"dataGaName":326,"dataGaLocation":27},"/company/team/","team",{"text":328,"config":329},"Handbook",{"href":330,"dataGaName":331,"dataGaLocation":27},"https://handbook.gitlab.com/","handbook",{"text":333,"config":334},"Investor relations",{"href":335,"dataGaName":336,"dataGaLocation":27},"https://ir.gitlab.com/","investor relations",{"text":338,"config":339},"Trust Center",{"href":340,"dataGaName":341,"dataGaLocation":27},"/security/","trust center",{"text":343,"config":344},"AI Transparency Center",{"href":345,"dataGaName":346,"dataGaLocation":27},"/ai-transparency-center/","ai transparency center",{"text":348,"config":349},"Newsletter",{"href":350,"dataGaName":351,"dataGaLocation":27},"/company/contact/","newsletter",{"text":353,"config":354},"Press",{"href":355,"dataGaName":356,"dataGaLocation":27},"/press/","press",{"text":358,"config":359,"lists":360},"Contact us",{"dataNavLevelOne":300},[361],{"items":362},[363,366,371],{"text":34,"config":364},{"href":36,"dataGaName":365,"dataGaLocation":27},"talk to sales",{"text":367,"config":368},"Get help",{"href":369,"dataGaName":370,"dataGaLocation":27},"/support/","get help",{"text":372,"config":373},"Customer portal",{"href":374,"dataGaName":375,"dataGaLocation":27},"https://customers.gitlab.com/customers/sign_in/","customer portal",{"close":377,"login":378,"suggestions":385},"Close",{"text":379,"link":380},"To search repositories and projects, login to",{"text":381,"config":382},"gitlab.com",{"href":41,"dataGaName":383,"dataGaLocation":384},"search login","search",{"text":386,"default":387},"Suggestions",[388,390,394,396,400,404],{"text":56,"config":389},{"href":61,"dataGaName":56,"dataGaLocation":384},{"text":391,"config":392},"Code Suggestions (AI)",{"href":393,"dataGaName":391,"dataGaLocation":384},"/solutions/code-suggestions/",{"text":108,"config":395},{"href":110,"dataGaName":108,"dataGaLocation":384},{"text":397,"config":398},"GitLab on AWS",{"href":399,"dataGaName":397,"dataGaLocation":384},"/partners/technology-partners/aws/",{"text":401,"config":402},"GitLab on Google Cloud",{"href":403,"dataGaName":401,"dataGaLocation":384},"/partners/technology-partners/google-cloud-platform/",{"text":405,"config":406},"Why GitLab?",{"href":69,"dataGaName":405,"dataGaLocation":384},{"freeTrial":408,"mobileIcon":413,"desktopIcon":418,"secondaryButton":421},{"text":409,"config":410},"Start free trial",{"href":411,"dataGaName":32,"dataGaLocation":412},"https://gitlab.com/-/trials/new/","nav",{"altText":414,"config":415},"Gitlab Icon",{"src":416,"dataGaName":417,"dataGaLocation":412},"https://res.cloudinary.com/about-gitlab-com/image/upload/v1758203874/jypbw1jx72aexsoohd7x.svg","gitlab icon",{"altText":414,"config":419},{"src":420,"dataGaName":417,"dataGaLocation":412},"https://res.cloudinary.com/about-gitlab-com/image/upload/v1758203875/gs4c8p8opsgvflgkswz9.svg",{"text":422,"config":423},"Get Started",{"href":424,"dataGaName":425,"dataGaLocation":412},"https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/compare/gitlab-vs-github/","get started",{"freeTrial":427,"mobileIcon":431,"desktopIcon":433},{"text":428,"config":429},"Learn more about GitLab Duo",{"href":61,"dataGaName":430,"dataGaLocation":412},"gitlab duo",{"altText":414,"config":432},{"src":416,"dataGaName":417,"dataGaLocation":412},{"altText":414,"config":434},{"src":420,"dataGaName":417,"dataGaLocation":412},{"freeTrial":436,"mobileIcon":441,"desktopIcon":443},{"text":437,"config":438},"Back to pricing",{"href":189,"dataGaName":439,"dataGaLocation":412,"icon":440},"back to pricing","GoBack",{"altText":414,"config":442},{"src":416,"dataGaName":417,"dataGaLocation":412},{"altText":414,"config":444},{"src":420,"dataGaName":417,"dataGaLocation":412},"content:shared:en-us:main-navigation.yml","Main Navigation","shared/en-us/main-navigation.yml","shared/en-us/main-navigation",{"_path":450,"_dir":21,"_draft":6,"_partial":6,"_locale":7,"title":451,"button":452,"image":457,"config":461,"_id":463,"_type":13,"_source":15,"_file":464,"_stem":465,"_extension":18},"/shared/en-us/banner","is now in public beta!",{"text":453,"config":454},"Try the Beta",{"href":455,"dataGaName":456,"dataGaLocation":27},"/gitlab-duo/agent-platform/","duo banner",{"altText":458,"config":459},"GitLab Duo Agent Platform",{"src":460},"https://res.cloudinary.com/about-gitlab-com/image/upload/v1753720689/somrf9zaunk0xlt7ne4x.svg",{"layout":462},"release","content:shared:en-us:banner.yml","shared/en-us/banner.yml","shared/en-us/banner",{"_path":467,"_dir":21,"_draft":6,"_partial":6,"_locale":7,"data":468,"_id":672,"_type":13,"title":673,"_source":15,"_file":674,"_stem":675,"_extension":18},"/shared/en-us/main-footer",{"text":469,"source":470,"edit":476,"contribute":481,"config":486,"items":491,"minimal":664},"Git is a trademark of Software Freedom Conservancy and our use of 'GitLab' is under license",{"text":471,"config":472},"View page source",{"href":473,"dataGaName":474,"dataGaLocation":475},"https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/","page source","footer",{"text":477,"config":478},"Edit this page",{"href":479,"dataGaName":480,"dataGaLocation":475},"https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/-/blob/main/content/","web ide",{"text":482,"config":483},"Please contribute",{"href":484,"dataGaName":485,"dataGaLocation":475},"https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/-/blob/main/CONTRIBUTING.md/","please contribute",{"twitter":487,"facebook":488,"youtube":489,"linkedin":490},"https://twitter.com/gitlab","https://www.facebook.com/gitlab","https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg","https://www.linkedin.com/company/gitlab-com",[492,515,571,600,634],{"title":45,"links":493,"subMenu":498},[494],{"text":495,"config":496},"DevSecOps platform",{"href":54,"dataGaName":497,"dataGaLocation":475},"devsecops platform",[499],{"title":187,"links":500},[501,505,510],{"text":502,"config":503},"View plans",{"href":189,"dataGaName":504,"dataGaLocation":475},"view plans",{"text":506,"config":507},"Why Premium?",{"href":508,"dataGaName":509,"dataGaLocation":475},"/pricing/premium/","why premium",{"text":511,"config":512},"Why Ultimate?",{"href":513,"dataGaName":514,"dataGaLocation":475},"/pricing/ultimate/","why ultimate",{"title":516,"links":517},"Solutions",[518,523,525,527,532,537,541,544,548,553,555,558,561,566],{"text":519,"config":520},"Digital transformation",{"href":521,"dataGaName":522,"dataGaLocation":475},"/topics/digital-transformation/","digital transformation",{"text":133,"config":524},{"href":135,"dataGaName":133,"dataGaLocation":475},{"text":122,"config":526},{"href":104,"dataGaName":105,"dataGaLocation":475},{"text":528,"config":529},"Agile development",{"href":530,"dataGaName":531,"dataGaLocation":475},"/solutions/agile-delivery/","agile delivery",{"text":533,"config":534},"Cloud transformation",{"href":535,"dataGaName":536,"dataGaLocation":475},"/topics/cloud-native/","cloud transformation",{"text":538,"config":539},"SCM",{"href":118,"dataGaName":540,"dataGaLocation":475},"source code management",{"text":108,"config":542},{"href":110,"dataGaName":543,"dataGaLocation":475},"continuous integration & delivery",{"text":545,"config":546},"Value stream management",{"href":162,"dataGaName":547,"dataGaLocation":475},"value stream management",{"text":549,"config":550},"GitOps",{"href":551,"dataGaName":552,"dataGaLocation":475},"/solutions/gitops/","gitops",{"text":172,"config":554},{"href":174,"dataGaName":175,"dataGaLocation":475},{"text":556,"config":557},"Small business",{"href":179,"dataGaName":180,"dataGaLocation":475},{"text":559,"config":560},"Public sector",{"href":184,"dataGaName":185,"dataGaLocation":475},{"text":562,"config":563},"Education",{"href":564,"dataGaName":565,"dataGaLocation":475},"/solutions/education/","education",{"text":567,"config":568},"Financial services",{"href":569,"dataGaName":570,"dataGaLocation":475},"/solutions/finance/","financial services",{"title":192,"links":572},[573,575,577,579,582,584,586,588,590,592,594,596,598],{"text":204,"config":574},{"href":206,"dataGaName":207,"dataGaLocation":475},{"text":209,"config":576},{"href":211,"dataGaName":212,"dataGaLocation":475},{"text":214,"config":578},{"href":216,"dataGaName":217,"dataGaLocation":475},{"text":219,"config":580},{"href":221,"dataGaName":581,"dataGaLocation":475},"docs",{"text":242,"config":583},{"href":244,"dataGaName":245,"dataGaLocation":475},{"text":237,"config":585},{"href":239,"dataGaName":240,"dataGaLocation":475},{"text":247,"config":587},{"href":249,"dataGaName":250,"dataGaLocation":475},{"text":260,"config":589},{"href":262,"dataGaName":263,"dataGaLocation":475},{"text":252,"config":591},{"href":254,"dataGaName":255,"dataGaLocation":475},{"text":265,"config":593},{"href":267,"dataGaName":268,"dataGaLocation":475},{"text":270,"config":595},{"href":272,"dataGaName":273,"dataGaLocation":475},{"text":275,"config":597},{"href":277,"dataGaName":278,"dataGaLocation":475},{"text":280,"config":599},{"href":282,"dataGaName":283,"dataGaLocation":475},{"title":298,"links":601},[602,604,606,608,610,612,614,618,623,625,627,629],{"text":305,"config":603},{"href":307,"dataGaName":300,"dataGaLocation":475},{"text":310,"config":605},{"href":312,"dataGaName":313,"dataGaLocation":475},{"text":318,"config":607},{"href":320,"dataGaName":321,"dataGaLocation":475},{"text":323,"config":609},{"href":325,"dataGaName":326,"dataGaLocation":475},{"text":328,"config":611},{"href":330,"dataGaName":331,"dataGaLocation":475},{"text":333,"config":613},{"href":335,"dataGaName":336,"dataGaLocation":475},{"text":615,"config":616},"Sustainability",{"href":617,"dataGaName":615,"dataGaLocation":475},"/sustainability/",{"text":619,"config":620},"Diversity, inclusion and belonging (DIB)",{"href":621,"dataGaName":622,"dataGaLocation":475},"/diversity-inclusion-belonging/","Diversity, inclusion and belonging",{"text":338,"config":624},{"href":340,"dataGaName":341,"dataGaLocation":475},{"text":348,"config":626},{"href":350,"dataGaName":351,"dataGaLocation":475},{"text":353,"config":628},{"href":355,"dataGaName":356,"dataGaLocation":475},{"text":630,"config":631},"Modern Slavery Transparency Statement",{"href":632,"dataGaName":633,"dataGaLocation":475},"https://handbook.gitlab.com/handbook/legal/modern-slavery-act-transparency-statement/","modern slavery transparency statement",{"title":635,"links":636},"Contact Us",[637,640,642,644,649,654,659],{"text":638,"config":639},"Contact an expert",{"href":36,"dataGaName":37,"dataGaLocation":475},{"text":367,"config":641},{"href":369,"dataGaName":370,"dataGaLocation":475},{"text":372,"config":643},{"href":374,"dataGaName":375,"dataGaLocation":475},{"text":645,"config":646},"Status",{"href":647,"dataGaName":648,"dataGaLocation":475},"https://status.gitlab.com/","status",{"text":650,"config":651},"Terms of use",{"href":652,"dataGaName":653,"dataGaLocation":475},"/terms/","terms of use",{"text":655,"config":656},"Privacy statement",{"href":657,"dataGaName":658,"dataGaLocation":475},"/privacy/","privacy statement",{"text":660,"config":661},"Cookie preferences",{"dataGaName":662,"dataGaLocation":475,"id":663,"isOneTrustButton":90},"cookie preferences","ot-sdk-btn",{"items":665},[666,668,670],{"text":650,"config":667},{"href":652,"dataGaName":653,"dataGaLocation":475},{"text":655,"config":669},{"href":657,"dataGaName":658,"dataGaLocation":475},{"text":660,"config":671},{"dataGaName":662,"dataGaLocation":475,"id":663,"isOneTrustButton":90},"content:shared:en-us:main-footer.yml","Main Footer","shared/en-us/main-footer.yml","shared/en-us/main-footer",{"allPosts":677,"featuredPost":4781,"totalPagesCount":4800,"initialPosts":4801},[678,705,727,751,772,796,817,836,858,879,900,920,942,963,983,1003,1023,1043,1064,1085,1107,1128,1148,1169,1188,1206,1227,1248,1271,1292,1313,1333,1353,1374,1395,1415,1435,1453,1473,1493,1514,1533,1553,1573,1593,1615,1634,1656,1676,1696,1716,1735,1755,1774,1791,1811,1830,1848,1867,1884,1905,1925,1944,1962,1981,2000,2018,2037,2057,2076,2095,2114,2133,2152,2171,2192,2212,2233,2254,2274,2295,2316,2335,2355,2373,2392,2410,2429,2449,2468,2488,2507,2527,2547,2565,2586,2608,2627,2647,2667,2687,2707,2728,2747,2767,2788,2807,2826,2845,2864,2884,2903,2922,2941,2960,2980,2998,3018,3037,3056,3076,3095,3117,3136,3155,3175,3195,3214,3234,3254,3272,3290,3309,3326,3345,3365,3384,3403,3423,3442,3462,3481,3501,3521,3543,3562,3581,3600,3619,3638,3656,3676,3693,3712,3732,3749,3768,3788,3807,3826,3845,3864,3882,3901,3921,3941,3960,3978,3994,4011,4031,4048,4066,4082,4101,4120,4138,4157,4177,4196,4216,4235,4254,4273,4292,4311,4330,4349,4368,4386,4406,4425,4443,4462,4480,4498,4519,4537,4556,4574,4592,4611,4631,4650,4668,4686,4704,4722,4741,4762],{"_path":679,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":680,"content":688,"config":698,"_id":701,"_type":13,"title":702,"_source":15,"_file":703,"_stem":704,"_extension":18},"/en-us/blog/4-ways-to-accelerate-embedded-development-with-gitlab",{"title":681,"description":682,"ogTitle":681,"ogDescription":682,"noIndex":6,"ogImage":683,"ogUrl":684,"ogSiteName":685,"ogType":686,"canonicalUrls":684,"schema":687},"4 ways to accelerate embedded development with GitLab","Learn how automated hardware testing, standard builds, collaborative workflows, and integrated compliance eliminate bottlenecks in firmware development.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659756/Blog/Hero%20Images/REFERENCE_-_display_preview_for_blog_images.png","https://about.gitlab.com/blog/4-ways-to-accelerate-embedded-development-with-gitlab","https://about.gitlab.com","article","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"4 ways to accelerate embedded development with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Matt DeLaney\"},{\"@type\":\"Person\",\"name\":\"Darwin Sanoy\"}],\n        \"datePublished\": \"2025-06-05\",\n      }",{"title":681,"description":682,"authors":689,"heroImage":683,"date":692,"body":693,"category":694,"tags":695},[690,691],"Matt DeLaney","Darwin Sanoy","2025-06-05","Software in embedded systems is no longer just a part number — it's a critical differentiator. This shift has led to enormous complexity in the firmware running in our cars, airplanes, and industrial machines. The number of lines of code in the average car is expected to reach [650 million](https://www.statista.com/statistics/1370978/automotive-software-average-lines-of-codes-per-vehicle-globally/) by the end of 2025, up from 200 million just five years ago. In aerospace systems, the complexity of embedded software has nearly [doubled every four years](https://www.mckinsey.com/industries/aerospace-and-defense/our-insights/debugging-the-software-talent-gap-in-aerospace-and-defense) for the last several decades. \n\nTraditional embedded development approaches cannot effectively handle the software challenges of modern machines. This shortcoming slows engineers down, in part, by exacerbating challenges such as: \n\n* [Hardware testing bottlenecks](#challenge-1-hardware-testing-bottlenecks) \n* [Inconsistent build environments](#challenge-2-inconsistent-build-environments)\n* [Siloed development practices](#challenge-3-siloed-development-practices)\n* [Manual functional safety compliance processes](#challenge-4-manual-functional-safety-compliance-processes)\n\nEmbedded developers need a new approach to deal with the rapid increase in code. In this article, we’ll explain four ways you can use the GitLab AI-native DevSecOps platform to shorten feedback loops, work collaboratively and iteratively, and streamline compliance.\n\n## Challenge 1: Hardware testing bottlenecks\n\nUnlike enterprise software that can run on virtually any cloud server, embedded automotive software must be tested on specialized hardware that precisely matches production environments. Traditional hardware-in-the-loop (HIL) testing processes often follow this pattern:\n\n1. Developers write code for an embedded system (e.g., an electronic control unit)  \n2. They request access to limited, expensive hardware test benches (costing $500,000-$10M each)  \n3. They wait days or weeks for their scheduled access window  \n4. They manually deploy and test their code on physical hardware at their desks  \n5. They document results, pass the hardware to the next developer, and go to the back of the hardware testing queue\n\nThis process is extremely inefficient. Embedded developers may finish writing their code today and wait weeks to test it on a hardware target. By then, they've moved on to other tasks. This context switching drains productivity. Not only that, developers may wait weeks to learn they had a simple math error in their code. \n\n### Solution: Automated hardware allocation and continuous integration\n\nYou can streamline hardware testing through automation using the [GitLab On-Premises Device Cloud](https://gitlab.com/guided-explorations/embedded/ci-components/device-cloud), a CI/CD component. This lets you automate the orchestration of scarce hardware resources, turning a manual, time-intensive process into a streamlined, continuous workflow.\n\nThe On-Premises Device Cloud:\n\n1. Creates pools of shared hardware resources  \n2. Automatically — and exclusively — allocates hardware to a developer’s hardware testing pipeline tasks based on availability  \n3. Deploys and executes tests without manual intervention  \n4. Collects and reports results through integrated pipelines  \n5. Automatically deallocates hardware back into the “available” pool\n\nAfter submitting code, you’ll receive results in hours instead of days, often without ever physically touching the test hardware.\n\nWhat this video for an introduction to the GitLab On-Premises Device Cloud CI/CD Component to orchestrate the remote allocation of shared hardware for HIL:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ltr2CIM9Zag?si=NOij3t1YYz4zKajC\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nYou can also adopt multi-pronged testing strategies that balance speed and quality. Bring the following embedded test patterns and environments into automated GitLab CI pipelines:\n\n* **Software-in-the-loop (SIL):** Testing on virtual hardware simulators for quicker initial feedback  \n* **Processor-in-the-loop (PIL):** Testing on representative processor hardware for faster feedback at a lower cost  \n* **Hardware-in-the-loop (HIL):** Testing on full production-equivalent hardware and test benches for late-stage verification\n\nBy automating the orchestration of these tests within CI pipelines, you’ll be able to identify issues earlier, iterate faster, and accelerate time to market.\n\n## Challenge 2: Inconsistent build environments\n\nAnother significant challenge in embedded development is build environment inconsistency. Embedded developers often manually execute builds on their local machines with varying configurations, compiler versions, and dependencies. Then they’ll paste the binaries from their local build to a shared codebase.\n\nThis approach creates several problems:\n\n* **Inconsistent outputs:** Builds for the same source code produce different results on different machines  \n* **\"Works on my machine\" syndrome:** Code that builds locally fails in shared environments  \n* **Poor traceability:** Limited audit trail of who built what and when  \n* **Knowledge silos:** Build expertise becomes concentrated in a few individuals\n\nThis approach can lead to errors, bottlenecks, and costly delays. \n\n### Solution: Standardized build automation\n\nYou can address these challenges by implementing standardized build automation within CI/CD pipelines in GitLab. This approach creates consistent, repeatable, container-based build environments that eliminate machine-specific variations. Through the use of special Embedded Gateway Runner provisioning scripts, containers can interface with hardware for flashing and port monitoring for automated testing.\n\nKey elements of this solution include:\n\n* **Lifecycle managed environments:** Define complex embedded simulation environments as code; automatically deploy environments for testing and destroy them afterward  \n* **Containerization:** Use Docker containers to ensure identical build environments  \n* **Automated dependency management:** Control and version all dependencies  \n* **Central build execution:** Run builds on shared infrastructure rather than local machines\n\n> Follow this tutorial to learn [how to automate embedded software builds within a GitLab CI pipeline](https://gitlab.com/guided-explorations/embedded/workshops/embedded-devops-workshop-refactoring-to-ci/-/blob/main/TUTORIAL2.md%20).\n\nBy standardizing and automating the build process, you can ensure that every build follows the same steps with the same dependencies, producing consistent outputs regardless of who initiated it. This not only improves quality but also democratizes the build process, enabling more team members to participate without specialized knowledge.\n\n## Challenge 3: Siloed development practices\n\nEnterprise development teams have widely adopted collaborative practices such as DevOps, underpinned by shared source code management (SCM) and continuous integration/continuous delivery (CI/CD) systems. Embedded developers, on the other hand, have historically worked alone at their desks. There are valid technical reasons for this. \n\nFor example, consider hardware virtualization, which is a key enabler of DevOps automation. The industry has been slower to virtualize the massive range of specialized processors and boards used in embedded systems. This is due in large part to the difficulties of virtualizing production real-time systems and the associated lack of economic incentives. Compare that to cloud virtualization which has been commoditized and benefited enterprise SaaS development for over a decade.\n\nMany providers are now embracing virtualization-first for the sake of speeding up embedded development. If teams fail to adopt virtual testing options, however, their silos will remain and negatively impact the business through: \n\n* **Knowledge fragmentation**: Critical insights remain scattered across individuals and teams  \n* **Redundant development**: Multiple teams solve identical problems, creating inconsistencies  \n* **Late-stage discovery during big-bang integrations**: Problems are found late in the process when multiple developers integrate their code at once, when errors are more costly to fix  \n* **Stifled innovation**: Solutions from one domain rarely influence others, hampering the development of new product ideas\n\n### Solution: Collaborative engineering through a unified platform\n\nAn important step in breaking down these silos is to standardize embedded development around GitLab’s unified DevSecOps platform. In this regard, GitLab is aligned with the shift of embedded systems toward more consolidated, shared platforms on embedded devices. GitLab enables:\n\n* **Shared visibility:** Make all code, Issues, and documentation visible across teams  \n* **Collaborative workflows:** Enable peer review and knowledge sharing through merge requests  \n* **Centralized knowledge:** Maintain a single source of truth for all development artifacts  \n* **Asynchronous collaboration:** Allow teams to work together across different locations and time zones\n\nHuman-AI agent collaboration is a fundamental ingredient to fueling the customer-facing innovations that digital natives and established embedded brands desire. GitLab enables human-AI collaboration as well. By creating transparency across the development lifecycle, GitLab changes embedded development from an isolated activity to a collaborative practice. Engineers can see each other's work in progress, learn from collective experiences, and build upon shared solutions.\n\nWatch this presentation from Embedded World Germany 2025, which explains the power of embedded developers collaborating and sharing “work in progress”. The demo portion from 24:42 to 36:51 shows how to integrate HIL into a GitLab CI pipeline and enable collaborative development.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/F_rlOyq0hzc?si=eF4alDY6HK98uZPj\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nPerhaps most importantly, by achieving greater collaboration through DevSecOps, teams can unlock embedded systems innovations that would otherwise remain hidden. Indeed, collaboration fuels innovation. [One study](https://www.sciencedirect.com/science/article/abs/pii/S0749597800928887), for example, found that group brainstorming, when properly structured, can lead to more innovative and creative outcomes than individuals working alone. Collaborative development is crucial in the race to develop software-defined products. \n\n## Challenge 4: Manual functional safety compliance processes\n\nEmbedded systems in the automotive and aerospace industries must comply with rigorous functional safety standards, including ISO 26262, MISRA C/C++, DO-178C, and DO-254. Traditional compliance approaches involve manual reviews, extensive documentation, and separate verification activities that occur late in the development cycle. This often creates security review bottlenecks. When specialized embedded security and code quality scanners detect vulnerabilities in a developer’s code, the scan issue gets added to a pile of other issues that haven’t been resolved. Developers can’t integrate their code, and security personnel need to wade through a backlog of code violations. This creates delays and makes compliance more difficult. \n\nSome of the challenges can best be summed up as: \n\n* **Late-stage compliance issues**: Problems discovered after development is complete  \n* **Documentation burden**: Extensive manual effort to create and maintain compliance evidence  \n* **Process bottlenecks**: Serial compliance activities that block development progress  \n* **Expertise dependence**: Reliance on limited specialists for compliance activities\n\nAs a result, teams often need to choose between velocity and compliance — a precarious trade-off in safety-critical systems.\n\n### Solution: Automated functional safety compliance workflow building blocks\n\nRather than treating security and compliance as post-development verification activities, you can codify compliance requirements and enforce them automatically through [customizable frameworks in GitLab](https://about.gitlab.com/blog/introducing-custom-compliance-frameworks-in-gitlab/). To do this for functional safety standards, in particular, you can integrate GitLab with specialized embedded tools, which provide the depth of firmware scanning required by functional safety standards. Meanwhile, GitLab provides automated compliance checks, full audit trails, and merge request gating — all features needed to support a robust continuous compliance program. \n\nThis integrated approach includes:\n\n* **Compliance-as-code:** Define compliance requirements as automated checks  \n* **Integrated specialized tools:** Connect tools like CodeSonar into the DevSecOps platform for automotive-specific compliance  \n* **Continuous compliance verification:** Verify requirements throughout development  \n* **Automated evidence collection:** Gather compliance artifacts as a by-product of development\n\nWatch this video to learn how to use Custom Compliance Frameworks in GitLab to create your own compliance policies. You can create compliance policies related to any standard (e.g., ISO 26262) and automatically enforce those policies in GitLab.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/S-FQjzSyVJw?si=0UdtGNuugLPG0SLL\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nBy shifting compliance left and embedding it within normal development workflows, you can maintain safety standards without sacrificing velocity. Automated checks catch issues early when they're easier and less expensive to fix, while continuous evidence collection reduces the documentation burden.\n\n## Realizing the power of embedded DevOps\n\nEmbedded development is changing fast. Teams that remain stuck in manual processes and isolated workflows will find themselves increasingly left behind, while those that embrace automated, collaborative practices will define the future of software-defined smart systems.\n\nExplore our [Embedded DevOps Workshop](https://gitlab.com/guided-explorations/embedded/workshops/embedded-devops-workshop-refactoring-to-ci) to start automating embedded development workflows with GitLab, or [watch this presentation from GitLab's Field Chief Cloud Architect](https://content.gitlab.com/viewer/0a35252831bd130f879b0725738f70ed) to learn how leading organizations are bringing hardware-in-the-loop testing into continuous integration workflows to accelerate embedded development.\n\n## Learn more\n\n- [Why GitLab Premium with Duo for embedded systems development?](https://content.gitlab.com/viewer/438451cba726dd017da7b95fd0fb1b59)\n- [Why GitLab Ultimate with Duo for embedded systems development?](https://content.gitlab.com/viewer/87f5104c26720e2c0d73a6b377522a44)\n- [More embedded development systems presentations from GitLab](https://content.gitlab.com/viewer/e59c40099d5e3c8f9307afb27c4a923f)","product",[495,9,696,697],"features","embedded DevOps",{"slug":699,"featured":6,"template":700},"4-ways-to-accelerate-embedded-development-with-gitlab","BlogPost","content:en-us:blog:4-ways-to-accelerate-embedded-development-with-gitlab.yml","4 Ways To Accelerate Embedded Development With Gitlab","en-us/blog/4-ways-to-accelerate-embedded-development-with-gitlab.yml","en-us/blog/4-ways-to-accelerate-embedded-development-with-gitlab",{"_path":706,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":707,"content":713,"config":721,"_id":723,"_type":13,"title":724,"_source":15,"_file":725,"_stem":726,"_extension":18},"/en-us/blog/5-videos-and-interactive-tours-to-learn-gitlab-duo-fast",{"title":708,"description":709,"ogTitle":708,"ogDescription":709,"noIndex":6,"ogImage":710,"ogUrl":711,"ogSiteName":685,"ogType":686,"canonicalUrls":711,"schema":712},"5 videos and interactive tours to learn GitLab Duo fast","Get to know GitLab Duo's capabilities and benefits, and use these visual learning tools to understand how to incorporate AI throughout your software development lifecycle.\n","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659856/Blog/Hero%20Images/blog-hero-banner-1-0178-820x470-fy25.png","https://about.gitlab.com/blog/5-videos-and-interactive-tours-to-learn-gitlab-duo-fast","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"5 videos and interactive tours to learn GitLab Duo fast\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2024-08-28\",\n      }",{"title":708,"description":709,"authors":714,"heroImage":710,"date":716,"body":717,"category":718,"tags":719},[715],"Cesar Saavedra","2024-08-28","GitLab Duo is a suite of AI-powered features designed to assist DevSecOps teams throughout the software development lifecycle. Integrated seamlessly into the GitLab platform, GitLab Duo leverages artificial intelligence to enhance productivity, improve code quality, and streamline various development and security processes. This article introduces you to GitLab Duo's capabilities and benefits, and lists five videos and interactive tours to help you learn how to incorporate this AI powerhouse into your own workflow.\n\nIn this article:\n- [GitLab Duo features](#gitlab-duo-features)\n- [Benefits of GitLab Duo](#benefits-of-gitlab-duo)\n- [5 videos and interactive tours](#5-videos-and-interactive-tours-to-learn-gitlab-duo)\n\n## GitLab Duo features\n\n[GitLab Duo](https://about.gitlab.com/gitlab-duo/) offers a wide range of AI-powered capabilities to help you ship more secure software faster and deliver better results for your customers.\n\n### Feature development\n\n- **Code Suggestions:** Helps developers write code more efficiently by generating code and showing suggestions as they type.\n\n- **Chat:** A conversational interface that answers questions and assists with various tasks throughout the development process.\n\n- **Code Explanation:** Helps understand selected code by providing clear explanations.\n\n- **GitLab Duo for the CLI:** Helps discover or recall Git commands when and where you need them.\n\n-  **Merge Commit Message Generation:** Helps merge more quickly by generating meaningful commit messages.\n\n- **Test Generation:** Helps catch bugs early by automatically generating tests for selected code.\n\n### Securing applications\n\n- **Vulnerability Explanation:** Shows information about security vulnerabilities in code and explains how to fix them.\n\n- **Vulnerability Resolution:** Helps resolve a vulnerability by generating a merge request that addresses it. (Beta)\n\n### Facilitating collaboration\n\n- **AI Impact Dashboard:** Measures the effectiveness and impact of AI on software development lifecycle metrics.\n\n- **Code Review Summary:** Makes merge request handover to reviewers easier by summarizing all the comments in a merge request review. (Experimental)\n\n- **Discussion Summary:** Helps everyone get up to speed by summarizing lengthy conversations in an issue. (Beta)\n\n- **Issue Description Generation:** Helps populate an issue quickly by generating a more in-depth description based on a short summary. (Experimental)\n\n- **Merge Request Summary:** Helps populate a merge request more quickly by generating a description based on the code changes. (Beta)\n\n- **Product Analytics:** Processes and responds to questions about your application's usage data.\n\n### Advanced troubleshooting\n\n- **Root Cause Analysis:** Helps determine the cause of CI/CD job failures by analyzing logs.\n\nThese components work together to provide comprehensive AI-assisted support throughout the software development lifecycle.\n\n## Benefits of GitLab Duo\n\nGitLab Duo offers numerous benefits to development teams and organizations. By integrating AI-powered assistance throughout the development lifecycle, it helps increase productivity, improve code quality, and enhance security. \nDevelopers can write code faster, understand complex codebases more easily, and catch potential issues earlier in the development process.\n\nGitLab Duo also helps streamline collaboration, speed up code reviews, and provide valuable insights into the impact of AI on ROI metrics. These benefits contribute to faster delivery of high-quality, secure software.\n\n## 5 videos and interactive tours to learn GitLab Duo\n\nTo help you get acquainted with GitLab Duo and its capabilities quickly, we've compiled a list of five videos and interactive tours. These visual learning tools provide an in-depth look at an array of GitLab Duo features and demonstrate how they can be integrated into your development workflow.\n\n__1. GitLab Duo Overview__\n\nThis comprehensive video introduces the core concepts of GitLab Duo and showcases its integration within the GitLab platform.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/o2xmLTV1y0I?si=90yPCHS_x2zSBAqe\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n__2. Code Suggestions in Action__\n\nAn interactive tour demonstrating how GitLab Duo Code Suggestions works in real-time, helping developers write code more efficiently.\n\n\u003Ca href=\"https://gitlab.navattic.com/code-suggestions\">\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752175911/Blog/b5gdnls7jdyrpeyjby5j.png\" alt=\"GitLab Duo Code Suggestions cover image\">\u003C/a>\n\n__3. Vulnerability Resolution Walkthrough__\n\nThis video guide takes you through the process of using GitLab Duo to understand and resolve security vulnerabilities in your code.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/VJmsw_C125E?si=cUmRiQNJbrv5Yd9D\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n__4. Chat Demo__\n\nAn interactive session showing how developers can leverage GitLab Duo Chat to get answers, generate code, and solve problems throughout the development process.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"\nhttps://www.youtube.com/embed/RJezT5_V6dI?si=QomHCGUKstnAwplM\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n__5. AI Impact Dashboard Tutorial__\n\nA detailed look at how to use and interpret the AI Impact Dashboard to measure the effectiveness of GitLab Duo in your development processes.\n\n\u003Ca href=\"https://gitlab.navattic.com/ai-impact\">\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752175921/Blog/hn7gflmqswrjb33unuja.png\" alt=\"GitLab Duo AI Impact Dashboard cover image\">\u003C/a>\u003C/p>\n\n## Get started with GitLab Duo today\n\nThese videos and interactive tours offer practical insights into how GitLab Duo can enhance your development workflow. By exploring these resources, you'll gain a better understanding of how to leverage AI-powered assistance to improve productivity, code quality, and security in your projects.\n\n> #### Become a GitLab Duo expert: [Start your free trial today!](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/?type=free-trial&toggle=gitlab-duo-pro)\n\n## Read more\n- [10 best practices for using AI-powered GitLab Duo Chat](https://about.gitlab.com/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat/)\n- [Refactor code into modern languages with AI-powered GitLab Duo](https://about.gitlab.com/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo/)\n- [Developing GitLab Duo blog series](https://about.gitlab.com/blog/developing-gitlab-duo-series/)\n- [Mastering GitLab admin tasks with GitLab Duo Chat](https://about.gitlab.com/blog/mastering-gitlab-admin-tasks-with-gitlab-duo-chat/)","ai-ml",[720,696,9,694],"AI/ML",{"slug":722,"featured":6,"template":700},"5-videos-and-interactive-tours-to-learn-gitlab-duo-fast","content:en-us:blog:5-videos-and-interactive-tours-to-learn-gitlab-duo-fast.yml","5 Videos And Interactive Tours To Learn Gitlab Duo Fast","en-us/blog/5-videos-and-interactive-tours-to-learn-gitlab-duo-fast.yml","en-us/blog/5-videos-and-interactive-tours-to-learn-gitlab-duo-fast",{"_path":728,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":729,"content":735,"config":745,"_id":747,"_type":13,"title":748,"_source":15,"_file":749,"_stem":750,"_extension":18},"/en-us/blog/a-beginners-guide-to-the-git-reftable-format",{"title":730,"description":731,"ogTitle":730,"ogDescription":731,"noIndex":6,"ogImage":732,"ogUrl":733,"ogSiteName":685,"ogType":686,"canonicalUrls":733,"schema":734},"A beginner's guide to the Git reftable format","In Git 2.45.0, GitLab upstreamed the reftable backend to Git, which completely changes how references are stored. Get an in-depth look at the inner workings of this new format.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664595/Blog/Hero%20Images/blog-image-template-1800x945__9_.png","https://about.gitlab.com/blog/a-beginners-guide-to-the-git-reftable-format","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"A beginner's guide to the Git reftable format\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Patrick Steinhardt\"}],\n        \"datePublished\": \"2024-05-30\",\n      }",{"title":730,"description":731,"authors":736,"heroImage":732,"date":738,"body":739,"category":740,"tags":741},[737],"Patrick Steinhardt","2024-05-30","Until recently, the \"files\" format was the only way for Git to store references. With the [release of Git 2.45.0](https://about.gitlab.com/blog/whats-new-in-git-2-45-0/), Git can now store references in a \"reftable\" format. This new format is a binary format that is quite a bit more complex, but that complexity allows it to address several shortcomings of the \"files\" format. The design goals for the \"reftable\" format include:\n\n- Make the lookup of a single reference and iteration through ranges of references as efficient and fast as possible.\n- Support for consistent reads of references so that Git never reads an in-between state when an update to multiple references has been applied only partially.\n- Support for atomic writes such that updating multiple references can be implemented as an all-or-nothing operation.\n- Efficient storage of both refs and the reflog.\n\nIn this article, we will go under the hood of the \"reftable\" format to see exactly how it works.\n\n## How Git stores references\n\nBefore we dive into the details of the \"reftable\" format, let's quickly recap how Git has historically stored references. If you are already familiar with this, you can skip this section.\n\nA Git repository keeps track of two important data structures:\n\n- [Objects](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), which contain the actual data of your repository. This includes commits, the directory tree structure, and the blobs that contain your source code. Objects point to each other, forming an object graph. Furthermore, each object has an object ID that uniquely identifies the object.\n\n- References, such as branches and tags, which are pointers into the object graph so that you can give objects names that are easier to remember and keep track of different tracks of your development history. For example, a repository may contain a `main` branch, which is a reference named `refs/heads/main` that points to a specific commit.\n\nReferences are stored in the reference database. Until Git 2.45.0, there was only the \"files\" database format. In this format, every reference is stored as a normal file that contains either one of the following:\n\n- A regular reference that contains the object ID of the commit it points to.\n- A symbolic reference that contains the name of another reference, similar to how a symbolic link points to another file.\n\nAt regular intervals, these references get packed into a single `packed-refs` file to make lookups more efficient.\n\nThe following examples should give an idea of how the \"files\" format operates:\n\n```shell\n$ git init .\n$ git commit --allow-empty --message \"Initial commit\"\n[main (root-commit) 6917c17] Initial commit\n\n# HEAD is a symbolic reference pointing to refs/heads/main.\n$ cat .git/HEAD\nref: refs/heads/main\n\n# refs/heads/main is a regular reference pointing to a commit.\n$ cat .git/refs/heads/main\n6917c178cfc3c50215a82cf959204e9934af24c8\n\n# git-pack-refs(1) packs these references into the packed-refs file.\n$ git pack-refs --all\n$ cat .git/packed-refs\n# pack-refs with: peeled fully-peeled sorted\n6917c178cfc3c50215a82cf959204e9934af24c8 refs/heads/main\n```\n\n## High-level structure of reftables\n\nAssuming that you've got Git 2.45.0 or newer installed, you can create a repository with the \"reftable\" format by using the `--ref-format=reftable` switch:\n\n```shell\n$ git init --ref-format=reftable .\nInitialized empty Git repository in /tmp/repo/.git/\n$ git rev-parse --show-ref-format\nreftable\n\n# Irrelevant files have been removed for ease of understanding.\n$ tree .git\n.git\n├── config\n├── HEAD\n├── index\n├── objects\n├── refs\n│   └── heads\n└── reftable\n\t├── 0x000000000001-0x000000000002-40a482a9.ref\n\t└── tables.list\n\n4 directories, 6 files\n```\n\nFirst, looking at the repository configuration, you will see it has an `extension.refstorage` key:\n\n```shell\n$ cat .git/config\n[core]\n    repositoryformatversion = 1\n    filemode = true\n    bare = false\n    logallrefupdates = true\n[extensions]\n    refstorage = reftable\n```\n\nThis configuration indicates to Git that the repository has been initialized with the \"reftable\" format and tells Git to use the \"reftable\" backend to access it.\n\nWeirdly enough, the repository still has a few files that look as if the \"files\" backend was in use:\n\n- `HEAD` would usually be a symbolic reference pointing to your currently checked-out branch. While it is not used by the \"reftable\" backend, it is required for Git clients to detect the directory as a Git repository. Therefore, when using the \"reftable\" format, `HEAD` is a stub with contents `ref: refs/heads/.invalid`.\n\n- `refs/heads` is a file with contents `this repository uses the reftable format`. Git clients that do not know about the \"reftable\" format would usually expect this path to be a directory. Consequently, creating this path as a file intentionally causes such older Git clients to fail if they tried to access the repository with the \"files\" backend.\n\nThe actual references are stored in the `reftable/` directory:\n\n```shell\n$ tree .git/reftable\n.git/reftable/\n├── 0x000000000001-0x000000000001-794bd722.ref\n└── tables.list\n\n$ cat .git/reftable/tables.list\n0x000000000001-0x000000000001-794bd722.ref\n```\n\nThere are two files here:\n\n- `0x000000000001-0x000000000001-794bd722.ref` is a table containing references and the reflog data in a binary format.\n\n- `tables.list` is, well, a list of tables. In the current state of the repository, the file contains a single line, which is the name of the table. This file tracks the current set of active tables in the \"reftable\" database and is updated whenever new tables get added to the repository.\n\nUpdating a reference creates a new table:\n\n```shell\n$ git commit --allow-empty --message \"Initial commit\"\n[main (root-commit) 1472a58] Initial commit\n\n$ tree .git/reftable\n.git/reftable/\n├── 0x000000000001-0x000000000002-eb87d12b.ref\n└── tables.list\n\n$ cat .git/reftable/tables.list\n0x000000000001-0x000000000002-eb87d12b.ref\n```\n\nAs you can see, the previous table has been replaced with a new one. Furthermore, the `tables.list` file has been updated to contain the new table.\n\n## The structure of a table\n\nAs mentioned earlier, the actual data of the reference database is contained in tables. Roughly speaking, a table is split up into multiple sections:\n\n- The \"header\" contains metadata about the table. Along with some other information, this includes the version of the format, the block size, and the hash function used by the repository (for example, SHA1 or SHA256).\n- The \"ref\" section contains your references. These records have a key that equals the reference name and point to either an object ID for regular references, or to another reference for symbolic references.\n- The \"obj\" section contains reverse mapping from object IDs to the references that point to those object IDs. These allow Git to efficiently look up which references point to a given object ID.\n- The \"log\" section contains your reflog entries. These records have a key that equals the reference name plus an index that represents the number of the log entry. Furthermore, they contain the old and new object IDs as well as the message for that reflog entry.\n- The \"footer\" contains offsets to the various sections.\n\n![long table with all the reftable sections](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_1_-_Reftable_overview.svg)\n\nEach of the section types are structured in a similar manner. Sections contain a set of records that are sorted by each record's key. For example, when you have two ref records `refs/heads/aaaaa` and `refs/heads/bbb`, you have two ref records with these reference names as their respective keys, and `refs/heads/aaaaa` would come before `refs/heads/bbb`.\n\nFurthermore, each section is divided into blocks of a fixed length. This block length is encoded in the header and serves two purposes:\n\n- Given the start of the section as well as the block size, the reader implicitly knows where each of the blocks starts. This allows Git to easily seek into the middle of a section without reading preceding blocks, which enables binary searches over blocks to speed up the lookup of records.\n- It ensures that the reader knows how much data to read from the disk at a time. Consequently, the block size is by default set to 4KiB, which is the most common sector size for hard disks. The maximum block size is 16MB.\n\nWhen we peek into, for example, a \"ref\" section, it looks roughly like the following graphic. Note how its records are ordered lexicographically inside the blocks, but also across the blocks.\n\n![reference block uncompressed](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_2_-_Ref_block_uncompressed.svg)\n\nEquipped with the current information, we can locate a record by using the following steps:\n\n1. Perform a binary search over the blocks by looking at the keys of their respective first records, identifying the block that must contain our record.\n\n2. Perform a linear search over the records in that block.\n\nBoth of these steps are still somewhat inefficient. If we have many blocks we may have to read logarithmically many of them in our binary search to find the desired one. And when blocks contain many records, we potentially have to read all of them during the linear search.\n\nThe \"reftable\" format has additional built-in mechanisms to address these performance concerns. We will touch on these over the next few sections.\n\n### Prefix compression\n\nAs you may have noticed, all of the record keys share the same prefix `refs/`. This is a common thing in Git:\n\n- All branches start with `refs/heads/`.\n- All tags start with `refs/tags/`.\n\nTherefore, we expect that subsequent records will most likely share a significant prefix of their key. This is a good opportunity to save some precious disk space. Because we know that most keys will share a common prefix, it makes sense to optimize for this.\n\nThe optimization uses prefix compression. Every record encodes a prefix length that tells the reader how many bytes to reuse from the key of the preceding record. If we have two records, `refs/heads/a` and `refs/heads/b`, the latter can be encoded by specifying a prefix length of 11 and then only storing the suffix `b`. The reader will then take the first 11 bytes of `refs/heads/a`, which is `refs/heads/`, and append the suffix `b` to it.\n\n![prefix compression](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_3_-_Ref_block_prefix_compression.svg)\n\n### Restart points\n\nAs explained earlier, the best way to search for a reference in a block with our current understanding of the \"reftable\" format is to do a linear search. This is because records do not have a fixed length, so it is impossible for us to tell where records would start without scanning through the block from the beginning. Also, even if records were of fixed length, we would not be able to seek into the middle of a block because the prefix compression also requires us to read preceding records.\n\nDoing a linear search would be quite inefficient because blocks may contain hundreds or even thousands of records. To address this issue, the \"reftable\" format encodes so-called restart points into every block. Restart points are uncompressed records where the prefix compression is reset. Consequently, records at restart points always contain their full key and it becomes possible to directly seek to and read the record without having to read preceding records. These restart points are listed in the footer of each block.\n\nEquipped with this information, we can avoid performing a linear search over the block. Instead, we can now do a binary search over the restart points where we search for the first restart point with a key larger than the sought-after key. From there, it follows that the desired record must be located in the section spanning from the _preceding_ restart point to the identified one.\n\nThus, our initial procedure to look up a record (binary search for the block, linear search for the record) is now:\n\n1. Perform a binary search over the blocks, identifying the block that must contain our record.\n\n2. Perform a binary search over the restart points, identifying the sub-section of the block that must contain our record.\n\n3. Perform a linear search over the records in that sub-section.\n\n![Linear search for a record](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_4_-_Restart_points.svg)\n\n### Indices\n\nWhile the search for records inside a block is now reasonably efficient, it's still inefficient to locate the block itself. A binary search may be reasonably performant when you have a couple of blocks, but repositories with millions of references may have hundreds or even thousands of blocks. Without any additional data structure, this would cause logarithmically many disk seeks on average.\n\nTo avoid this, every section may be followed by an index section that provides an efficient way to look up a block. Each index record holds the following information:\n\n- The location of the block that it is indexing.\n- The key of the last record of the block that it is indexing.\n\nWith three or less blocks, a binary search will always require, at most, two disk reads to find the desired target block. This is the same number of reads we would have to do with an index: one to read the index itself and one to read the desired block. Consequently, indices are only written when they would actually save some reads, which is the case with four or more indexed blocks.\n\nNow the question is: What happens when the index itself becomes so large that it spans over multiple blocks? You might have guessed it: We write another index that indexes the index. These multi-level indices really only become necessary once you have repositories with hundreds of thousands of references.\n\nEquipped with these indices, we can now make the procedure to look up records even more efficient:\n1. Determine whether there is an index by looking at the footer of the table.\n\t- If there is one, perform a binary search over the index to find the desired block. This block may point into an index block itself, in which case we need to repeat this step until we hit a record of the desired type.\n\t- Otherwise, perform a binary search over the blocks as we did before.\n2. Perform a binary search over the restart points, identifying the sub-section of the block that must contain our record.\n3. Perform a linear search over the records in that sub-section.\n\n## Multiple tables\n\nUp to this point, we have only discussed how to read a _single_ table. But as the name `tables.list` indicates, you can actually have a list of tables in your \"reftable\" database.\n\nEvery time you update a reference in your repository, a new table is written and appended to `tables.list`. Thus, you will eventually end up with multiple tables:\n\n```shell\n$ tree .git/reftable/\n.git/reftable/\n├── 0x000000000001-0x000000000007-8dcd8a77.ref\n├── 0x000000000008-0x000000000008-30e0f6f6.ref\n└── tables.list\n\n$ cat .git/reftable/tables.list\n0x000000000001-0x000000000007-8dcd8a77.ref\n0x000000000008-0x000000000008-30e0f6f6.ref\n```\n\nReading the actual state of a repository requires us to merge these multiple tables into a single virtual table.\n\nYou might be wondering: If a table is written for each reference update and the same reference is updated multiple times, how does the \"reftable\" format know the most up-to-date value of a given reference? Intuitively, one could assume the value would be the one from the newest table containing the reference.\n\nIn fact, every single record has a so-called update index that encodes the \"priority\" of a record. For example, if two ref records with the same name exist, then the one with the higher update index overrides the one with the lower update index.\n\nThese update indices are visible in the file structure above. The long hex strings (for example `0x000000000001`) are the update indices, where the left-hand side of the table name is the minimum update index contained in the table and the right-hand is the maximum update index.\n\nMerging the tables then happens via a [priority queue](https://en.wikipedia.org/wiki/Priority_queue) that is ordered by the key of the ref record as well as its update index. Assuming we want to scan through all ref records, we would:\n\n1. For every table, add its first record to the priority queue.\n\n![Adding first record to the priority queue](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_5_-_Priority_queue_1.svg)\n\n2. Yield the head of the priority queue. Because the queue is ordered by update index, it must be the most up-to-date version. Add the next item from that table to the priority queue.\n\n![Yielding the head of the priority queue](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_6_-_Priority_queue_2.svg)\n\n3. Drop all records from the queue that have the same name. These records are shadowed, which means that they will not be shown. For each table for which we are dropping records, add the next record to the priority queue.\n\n![Dropping all records from queue that have the same name](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675179/Blog/Content%20Images/Frame_7_-_Priority_queue_3.svg)\n\nNow we can rinse and repeat to read records for other keys.\n\nTables may contain special \"tombstone\" records that mark a record as having been deleted. This allows us to delete records without having to rewrite all tables to not contain the record anymore.\n\n### Auto-compaction\n\nWhile the idea behind the priority queue is simple enough, it would be rather inefficient to merge together hundreds or even only dozens of tables in this way. So while it is true that every update to your references appends a new table to your `tables.list` file, it is only part of the story.\n\nThe other part is auto-compaction: After a new table has been appended to the list of tables, the \"reftable\" backend checks whether some of the tables should be merged. This is done by using a simple heuristic: We check whether the list of tables forms a [geometric sequence](https://en.wikipedia.org/wiki/Geometric_progression) with the file sizes. Every table `n` must be at least twice as large as the next-most-recent table `n + 1`. If that geometric sequence is violated, the backend will compact tables so that the geometric sequence is restored.\n\nOver time, this will lead to structures that look like the following:\n\n```shell\n$ du --apparent-size .git/reftable/*\n429    .git/reftable/0x000000000001-0x00000000bd7c-d9819000.ref\n101    .git/reftable/0x00000000bd7d-0x00000000c5ac-c34b88a4.ref\n32    .git/reftable/0x00000000c5ad-0x00000000cc6c-60391f53.ref\n8    .git/reftable/0x00000000cc6d-0x00000000cdc1-61c30db1.ref\n3    .git/reftable/0x00000000cdc2-0x00000000ce67-d9b55a96.ref\n1    .git/reftable/0x00000000ce68-0x00000000ce6b-44721696.ref\n1    .git/reftable/tables.list\n```\n\nNote how for every single table, the property `size(n) > size(n+1) * 2` holds.\n\nOne of the consequences of auto-compaction is that the \"reftable\" backend maintains itself. We no longer have to run `git pack-refs` in a repository.\n\n## Want to learn more?\n\nYou should now have a good understanding of how the new \"reftable\" format works under the hood. If you want to dive even deeper into the format, you can refer to the [technical documentation](https://git-scm.com/docs/reftable) provided by the Git project.\n\n> Read our [Git 2.45.0 recap](https://about.gitlab.com/blog/whats-new-in-git-2-45-0/) to find out what else is in this version of Git.","open-source",[742,9,743,744],"git","open source","performance",{"slug":746,"featured":90,"template":700},"a-beginners-guide-to-the-git-reftable-format","content:en-us:blog:a-beginners-guide-to-the-git-reftable-format.yml","A Beginners Guide To The Git Reftable Format","en-us/blog/a-beginners-guide-to-the-git-reftable-format.yml","en-us/blog/a-beginners-guide-to-the-git-reftable-format",{"_path":752,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":753,"content":759,"config":766,"_id":768,"_type":13,"title":769,"_source":15,"_file":770,"_stem":771,"_extension":18},"/en-us/blog/a-go-micro-language-framework-for-building-dsls",{"title":754,"description":755,"ogTitle":754,"ogDescription":755,"noIndex":6,"ogImage":756,"ogUrl":757,"ogSiteName":685,"ogType":686,"canonicalUrls":757,"schema":758},"Lingo: A Go micro language framework for building Domain Specific Languages","Design, build and integrate your own Domain Specific Language with Lingo.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682320/Blog/Hero%20Images/typeset.png","https://about.gitlab.com/blog/a-go-micro-language-framework-for-building-dsls","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Lingo: A Go micro language framework for building Domain Specific Languages\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Julian Thome\"}],\n        \"datePublished\": \"2022-05-26\",\n      }",{"title":754,"description":755,"authors":760,"heroImage":756,"date":762,"body":763,"category":740,"tags":764},[761],"Julian Thome","2022-05-26","\n\nDomain Specific Languages (DSL) are small, focused languages with a narrow\ndomain of applicability. DSLs are tailored towards their target domain so that\ndomain experts can formalize ideas based on their knowledge and background.\n\nThis makes DSLs powerful tools that can be used for the purpose of increasing\nprogrammer efficiency by being more expressive in their target\ndomain, compared to general purpose languages, and by providing concepts to\nreduce the cognitive load on their users.\n\nConsider the problem of summing up the balances of different bank accounts in a\nCSV file. A sample CSV file is provided in the example below where the first\ncolumn contains the name of the account holder and the second column contains\nthe account balance.\n\n``` csv\nname, balance\nLisa, 100.30\nBert, 241.41\nMaria, 151.13\n```\n\nYou could solve the problem of summing up balances by using a general-purpose\nlanguage such as [Ruby](https://www.ruby-lang.org/en/) as in the code snippet\nbelow. Apart from the fact that the code below is not very robust, it contains\na lot of boilerplate that is irrelevant to the problem at hand, i.e., summing\nup the account balances.\n\n``` ruby\n#!/usr/bin/env ruby\n\nexit(1) if ARGV.empty? || !File.exist?(ARGV[0])\n\nsum = 0\nFile.foreach(ARGV[0]).each_with_index do |line, idx|\n  next if idx == 0\n  sum += Float(line.split(',')[1])\nend\n\nputs sum.round(2)\n```\n\nBelow is an example [AWK script](https://en.wikipedia.org/wiki/AWK) that solves\nthe same problem. AWK is a DSL that was specifically designed to address\nproblems related to text-processing.\n\n``` awk \n#!/usr/bin/awk -f\n\nBEGIN{FS=\",\"}{sum+=$2}END{print sum}\n```\n\nThe Ruby program has a size of 208 characters, whereas the AWK program has a size of 56. The AWK program is roughly 4x smaller than its Ruby\ncounterpart. In addition, the AWK implementation is more robust by being less\nprone to glitches that may appear in the CSV file (e.g., empty newlines,\nwrongly formatted data-fields). The significant difference in terms of size\nillustrates that DSLs, by being more focused on solving specific problems, can\nmake their users more productive by sparing them the burden to write\nboilerplate code and narrowing the focus of the language on the problem at\nhand.\n\nSome popular DSLs most software developers use on a regular basis include\n[Regular Expressions](https://en.wikipedia.org/wiki/Regular_expression) for\npattern matching, AWK for text\ntransformation or [Standard Query Language](https://en.wikipedia.org/wiki/SQL)\nfor interacting with databases.\n\n## Challenges when designing Domain Specific Languages\n\nPrototyping, designing and evolving DSLs is a\nchallenging process. In our experience this is an exploratory cycle where you\nconstantly prototype ideas, incorporate them into the language, try them out in\nreality, collect feedback and improve the DSL based on the feedback. \n\nWhen designing a DSL, there are many components that have to be implemented and\nevolved. At a very high level there are two main components: the language\nlexer/parser and the language processor. The lexer/parser\nis the component that accepts input as per the language definition which is\nusually specified specified by means of a language grammar. The parsing/lexing\nphase produces a syntax tree which is then passed onto the language processor.\nA language processor evaluates the syntax tree. In the example we saw earlier,\nwe ran both the Ruby and AWK interpreters providing our scripts and the CSV\nfile as input; both interpreters evaluated the scripts and this evaluation\nyielded the sum of all the account balances as a result.\n\nTools such as parser generators can significantly reduce the effort of\nlexer/parser development by means of code generation. Sophisticated DSL\nframeworks such as [JetBrains MPS](https://www.jetbrains.com/mps/) or\n[Xtext](https://www.eclipse.org/Xtext/) also provide features that help\nimplement custom language support in IDEs. However, if present at all, the\nsupport for building the language processors is usually limited to generating\nplaceholders functions or boilerplate code for the language components that\nhave to be filled-in by the DSL developer. Moreover, such large and powerful DSL\nframeworks usually have a fairly steep learning curve so that they are probably\na better fit for more sophisticated DSLs as opposed to small, easily\nembeddable, focused languages, which we refer to as _micro languages_.\n\nIn some situations, it may be worth considering working around these problems\nby just relying on standard data exchange formats such as `.toml`, `.yaml` or\n`.json` as a means of configuration. Similar to the parser generators, using\nsuch a format may relieve some of the burden when it comes to parser\ndevelopment effort. However, this approach does not help when it comes to the\nimplementation of the actual language processor. In addition, most standard data\nexchange formats are inherently limited to representing data in terms of simple\nconcepts (such as lists, dictionaries, strings and numbers). This limitation\ncan lead to bloated configuration files quickly as shown in the following\nexample.\n\nImagine the development of a calculator that operates on integers using\nmultiplication `*`, addition `+`. When using a data-description language like\nYAML in the example below, you can see that even a small simple term like `1 + 2 * 3 + 5` \ncan be hard to reason about, and by adding more terms the configuration\nfile would get bloated quickly.\n\n``` yaml\nterm:\n  add: \n    - 1\n    - times:\n      - 2\n      - 3\n    - 5\n```\n\nThis blog post is focused on the design of micro languages. The core idea is to\nprovide a simple, extensible language core that can be easily extended with\ncustom-types and custom functions; the language can evolve without having\nto touch the parser or the language processor. Instead, the DSL designer can\njust focus on the concepts that ought to be integrated into the DSL by\nimplementing interfaces and \"hooking\" them into the core language\nimplementation.\n\n## Lingo: A micro language framework for Go\n\nAt GitLab, Go is one of our main programming languages and some of the tools we\ndevelop required their own, small, embeddable DSLs so that users could properly\nconfigure and interact with them. \n\nInitially, we tried to integrate already existing, embeddable and expandable\nlanguage implementations. Our only condition was that they had to be\nembeddable natively into a Go application. We explored several great free and\nopen-source (FOSS) projects such as [go-lua](https://github.com/Shopify/go-lua)\nwhich is Lua VM implemented in Go, [go-yeagi](https://github.com/traefik/yaegi)\nwhich provides a Go interpreter with which Go can be used as a scripting\nlanguage or [go-zygomys](https://github.com/glycerine/zygomys) which is a LISP\ninterpreter written in Go. However, these packages are essentially modules to\nintegrate general-purpose languages on top of which a DSL could be built. These modules ended up being fairly complex. In contrast, we wanted to have basic support to design, implement, embed and evolve DSLs natively into a Go\napplication that is flexible, small, simple/easy to grasp, evolve and\nadapt.\n\nWe were looking for a micro language framework with the properties listed below:\n\n1. Stability: Changes applied to the DSL should neither require any changes to the core lexer/parser implementation nor to the language processor implementation.\n1. Flexibility/Composability: New DSL concepts (data-types, functions) can be integrated via a simple plug-in mechanism.\n1. Simplicity: the language framework should have just\nenough features to provide a foundation that is powerful enough to implement\nand evolve a custom DSL. In addition, the whole implementation of the micro\nlanguage framework should be in pure Go so that it is easily embeddable in Go\napplications.\n\nSince none of the available FOSS tools we looked at was able to\nfulfill all of those requirements, we developed our own micro language framework\nin Go called Lingo which stands for \"**L**ISP-based Domain Specific Languages\n(DSLs) **in Go**\". Lingo is completely FOSS and available in the [Lingo Git repository](https://gitlab.com/gitlab-org/vulnerability-research/foss/lingo)\nunder the free and open source space of the [Vulnerability Research Team](https://about.gitlab.com/handbook/engineering/development/sec/secure/vulnerability-research/).\n\n[Lingo](https://gitlab.com/gitlab-org/vulnerability-research/foss/lingo)\nprovides a foundation for building DSLs based on Symbolic Expressions (S-expressions), i.e.,\nexpressions provided in the form of nested lists `(f ...)` where `f` can be\nconsidered as the placeholder that represents the function symbol. Using this format,\nthe mathematical term we saw earlier could be written as S-expression `(+ 1 (* 2 3) 5)`. \n\nS-expressions are versatile and easy to process due to their uniformity. In\naddition, they can be used to represent both code and data in a consistent\nmanner.\n\nWith regards to the Stability, Flexibility and Composability properties, \n[Lingo](https://gitlab.com/gitlab-org/vulnerability-research/foss/lingo)\nprovides a simple plug-in mechanism to add new functions as well as types\nwithout having to touch the core parser or language processor. From the\nperspective of the S-expression parser, the actual function symbol is\nessentially irrelevant with regards to the S-expression parsing. The language processor is just evaluating S-expressions and dispatching the execution to the interface implementations. These implementations are provided by the plug-ins based on the function symbol.\n\nWith regards to Simplicity, the Lingo code base is roughly 3K lines of pure Go code including the lexer/parser, an\nengine for code transformation, and the interpreter/evaluator. The small size\nshould make it possible to understand the entirety of the implementation.  \n\nReaders that are interested in the technical details of\nLingo itself can have a look at the\n[README.md](https://gitlab.com/gitlab-org/vulnerability-research/foss/lingo/-/blob/main/README.md)\nwhere the implementation details and the used theoretical foundations are explained.\nThis blog post focuses on how\nLingo can be used to build a DSL from scratch.\n\n## Using Lingo to design a data generation engine\n\nIn this example we are designing a data-generation engine in Go using\nLingo as a foundation. Our data generation engine may be used to generate structured input\ndata for fuzzing or other application contexts. This example illustrates how\nyou can use Lingo to create a language and the corresponding language\nprocessor. Going back to the example from the beginning, let us assume we would\nlike to generate CSV files in the format we saw at the beginning covering\naccount balances.\n\n``` csv\nname, balance\nLisa, 100.30\nBert, 241.41\nMaria, 151.13\n```\n\nOur language includes the following functions:\n\n1. `(oneof s0, s1, ..., sN)`: randomly returns one of the parameter strings `sX` (0 \u003C= X \u003C= N).\n1. `(join e0, e1, ..., eN)`: joins all argument expressions and concatenates their string representation `eX` (0 \u003C= X \u003C= N).\n1. `(genfloat min max)`: generates a random float number X (0 \u003C= X \u003C= N) and returns it.\n1. `(times num exp)`: repeats the pattern generated by exp num times.\n\nFor this example we are using\nLingo to build the language and the language processor to automatically generate CSV\noutput which we are going to feed back into the Ruby and AWK programs we saw in\nthe introduction in order to perform a stress test on them. \n\nWe refer to our new language/tool as _Random Text Generator_ (RTG) `.rtg`.\nBelow is a sample script `script.rtg` we'd like our program to digest in order\nto randomly generate CSV files. As you can see in the example below, we are\njoining sub-strings starting with the CSV header `name, balance`\nafter which we randomly generate 10 lines of names and balance amounts. In\nbetween, we also randomly generate some empty lines.\n\n```\n(join \n  (join \"name\" \",\" \"balance\" \"\\n\")\n  (times 10 \n    '(join \n      (oneof \n        \"Jim\" \n        \"Max\" \n        \"Simone\" \n        \"Carl\" \n        \"Paul\" \n        \"Karl\" \n        \"Ines\" \n        \"Jane\" \n        \"Geralt\" \n        \"Dandelion\" \n        \"Triss\" \n        \"Yennefer\" \n        \"Ciri\") \n      \",\" \n      (genfloat 0 10000) \n      \"\\n\" \n      (oneof \"\" \"\\n\"))))\n```\n\nOur engine takes the script above written in RTG and generates random CSV\ncontent. Below is an example CSV file generated from this script.\n\n``` csv\nname,balance\nCarl,25.648205\nInes,11758.551\n\nCiri,13300.558\n...\n```\n\nFor the remainder of this section, we explore how we can implement a\ndata generation engine based on Lingo. The implementation of RTG requires\nthe two main ingredients: (1) a float data type and a result object to integrate a float\nrepresentation and (2) implementations for the `times`, `oneof`, `genfloat` and\n`join` functions.\n\n### Introducing a float data type and result objects\n\nLingo differentiates between data types and result objects. Data types indicate how data is\nmeant to be used and result objects are used to pass intermediate results\nbetween functions where every result has a unique type. In the code snippet\nbelow, we introduce a new `float` data type. The comments in the code snippet below\nprovide more details.\n\n``` go \n// introduce float type\nvar TypeFloatId, TypeFloat = types.NewTypeWithProperties(\"float\", types.Primitive)\n// introduce token float type for parser\nvar TokFloat = parser.HookToken(parser.TokLabel(TypeFloat.Name))\n\n// recognize (true) as boolean\ntype FloatMatcher struct{}\n\n// this function is used by the parser to \"recognize\" floats as such\nfunc (i FloatMatcher) Match(s string) parser.TokLabel {\n  if !strings.Contains(s, \".\") {\n    return parser.TokUnknown\n  }\n\n  if _, err := strconv.ParseFloat(s, 32); err == nil {\n\treturn TokFloat.Label\n  }\n\n  return parser.TokUnknown\n}\nfunc (i FloatMatcher) Id() string {\n  return string(TokFloat.Label)\n}\n\nfunc init() {\n  // hook matcher into the parser\n  parser.HookMatcher(FloatMatcher{})\n}\n```\n\nIn addition, we also require a result object which we can use to pass around\nfloat values. This is an interface implementation where most of the functions names\nare self-explanatory. The important bit is the `Type` function\nthat returns our custom `float` type we introduced in the last snippet.\n\n``` go\ntype FloatResult struct{ Val float32 }\n// deep copy\nfunc (r FloatResult) DeepCopy() eval.Result { return NewFloatResult(r.Val) }\n// returns the string representation of this result type\nfunc (r FloatResult) String() string {\n  return strconv.FormatFloat(float64(r.Val), 'f', -1, 32)\n}\n// returns the data type for this result type\nfunc (r FloatResult) Type() types.Type   { return custtypes.TypeFloat }\n// call-back that is cleaned up when the environment is cleaned up\nfunc (r FloatResult) Tidy()              {}\n\nfunc (r FloatResult) Value() interface{} { return r.Val }\nfunc (r *FloatResult) SetValue(value interface{}) error {\n  boolVal, ok := value.(float32)\n  if !ok {\n    return fmt.Errorf(\"invalid type for Bool\")\n  }\n  r.Val = boolVal\n  return nil\n}\nfunc NewFloatResult(value float32) *FloatResult {\n  return &FloatResult{\n    value,\n  }\n}\n```\n\n### Implementing the DSL functions\n\nSimilar to the data type and return object, implementation of a DSL function is\nas simple as implementing an interface. In the example below we implement the\n`genfloat` function as an example. The most important parts are the `Symbol()`,\n`Validate()` and `Evaluate()` functions. The `Symbol()` function returns the\nfunction symbol which is `genfloat` in this particular case. \n\nBoth, the `Validate()` and `Evaluate()` functions take the environment `env`\nand the parameter Stack `stack` as the parameter. The environment is used to store\nintermediate results which is useful when declaring/defining variables. The `stack` includes the input parameters of the function. For\n`(genfloat 0 10000)`, the stack would consist out of two `IntResult` parameters\n`0` and `10000` where `IntResult` is a standard result object already provided by the\ncore implementation of Lingo. `Validate()` makes sure that the parameter can be\ndigested by the function at hand, whereas `Evaluate()` actually invokes the\nfunction. In this particular case, we are generating a float value within the\nspecified range and return the corresponding `FloatResult`.\n\n``` go\ntype FunctionGenfloat struct{}\n\n// returns a description of this function\nfunc (f *FunctionGenfloat) Desc() (string, string) {\n  return fmt.Sprintf(\"%s%s %s%s\",\n    string(parser.TokLeftPar),\n    f.Symbol(),\n\t\"min max\",\n\tstring(parser.TokRightPar)),\n\t\"generate float in rang [min max]\"\n}\n\n// this is the symbol f of the function (f ...)\nfunc (f *FunctionGenfloat) Symbol() parser.TokLabel {\n  return parser.TokLabel(\"genfloat\")\n}\n\n// validates the parameters of this function which are passed in\nfunc (f *FunctionGenfloat) Validate(env *eval.Environment, stack *eval.StackFrame) error {\n  if stack.Size() != 2 {\n    return eval.WrongNumberOfArgs(f.Symbol(), stack.Size(), 2)\n  }\n\n  for idx, item := range stack.Items() {\n    if item.Type() != types.TypeInt {\n\t  return eval.WrongTypeOfArg(f.Symbol(), idx+1, item)\n\t}\n  }\n  return nil\n}\n\n// evaluates the function and returns the result\nfunc (f *FunctionGenfloat) Evaluate(env *eval.Environment, stack *eval.StackFrame) (eval.Result, error) {\n  var result float32\n  rand.Seed(time.Now().UnixNano())\n  for !stack.Empty() {\n    max := stack.Pop().(*eval.IntResult)\n    min := stack.Pop().(*eval.IntResult)\n\n\tminval := float32(min.Val)\n\tmaxval := float32(max.Val)\n\n\tresult = minval + (rand.Float32() * (maxval - minval))\n  }\n\n  return custresults.NewFloatResult(result), nil\n}\n\nfunc NewFunctionGenfloat() (eval.Function, error) {\n  fun := &FunctionGenfloat{}\n  parser.HookToken(fun.Symbol())\n  return fun, nil\n}\n```\n\n### Putting it all together\n\nAfter implementing all the functions, we only have to register/integrate them\n(`eval.HookFunction(...)`) so that Lingo properly resolves them when processing\nthe program. In the example below, we are registering all of the custom functions\nwe implemented, i.e., `times`, `oneof`, `join`, `genfloat`. The `main()`\nfunction in the example below includes the code required to evaluate our script\n`script.rtg`.\n\n``` go\n// register function\nfunc register(fn eval.Function, err error) {\n  if err != nil {\n    log.Fatalf(\"failed to create %s function %s:\", fn.Symbol(), err.Error())\n  }\n  err = eval.HookFunction(fn)\n  if err != nil {\n    log.Fatalf(\"failed to hook bool function %s:\", err.Error())\n  }\n}\n\nfunc main() {\n  // register custom functions\n  register(functions.NewFunctionTimes())\n  register(functions.NewFunctionOneof())\n  register(functions.NewFunctionJoin())\n  register(functions.NewFunctionGenfloat())\n  register(functions.NewFunctionFloat())\n  if len(os.Args) \u003C= 1 {\n    fmt.Println(\"No script provided\")\n    os.Exit(1)\n  }\n  // evaluate script\n  result, err := eval.RunScriptPath(os.Args[1])\n  if err != nil {\n    fmt.Println(err.Error())\n    os.Exit(1)\n  }\n\n  // print output\n  fmt.Printf(strings.ReplaceAll(result.String(), \"\\\\n\", \"\\n\"))\n\n  os.Exit(0)\n}\n```\n\nThe source code for RTG is available\n[here](https://gitlab.com/julianthome/lingo-example). You can find information\nabout how to build and run the tool in the\n[README.md](https://gitlab.com/julianthome/lingo-example/-/blob/main/README.md).\n\nWith approx. 300 lines of Go code, we have successfully designed a language and\nimplemented a language processor. We can now use RTG to test the robustness of\nthe Ruby (`computebalance.rb`) and AWK scripts (`computebalance.awk`) we used\nat the beginning to sum up account balances. \n\n``` bash\ntimeout 10 watch -e './rtg script.rtg > out.csv && ./computebalance.awk out.csv'\ntimeout 10 watch -e './rtg script.rtg > out.csv && ./computebalance.rb out.csv'\n```\n\nThe experiment above shows that the files generated by means of RTG can be\nproperly digested by the AWK script which is much more robust since it can cope\nwith the all generated CSV files. In contrast, executing of the Ruby script\nresults in errors because it cannot properly cope with newlines as they appear\nin the CSV file.\n\nCover image by [Charles Deluvio](https://unsplash.com/@kristianstrand) on [Unsplash](https://unsplash.com/photos/p8gzCnZf39k)\n{: .note}\n\n",[765,9,696],"collaboration",{"slug":767,"featured":6,"template":700},"a-go-micro-language-framework-for-building-dsls","content:en-us:blog:a-go-micro-language-framework-for-building-dsls.yml","A Go Micro Language Framework For Building Dsls","en-us/blog/a-go-micro-language-framework-for-building-dsls.yml","en-us/blog/a-go-micro-language-framework-for-building-dsls",{"_path":773,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":774,"content":780,"config":790,"_id":792,"_type":13,"title":793,"_source":15,"_file":794,"_stem":795,"_extension":18},"/en-us/blog/a-visual-guide-to-gitlab-ci-caching",{"title":775,"description":776,"ogTitle":775,"ogDescription":776,"noIndex":6,"ogImage":777,"ogUrl":778,"ogSiteName":685,"ogType":686,"canonicalUrls":778,"schema":779},"A visual guide to GitLab CI/CD caching","Learn cache types, as well as when and how to use them.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682443/Blog/Hero%20Images/cover.jpg","https://about.gitlab.com/blog/a-visual-guide-to-gitlab-ci-caching","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"A visual guide to GitLab CI/CD caching\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Matthieu Fronton\"}],\n        \"datePublished\": \"2022-09-12\",\n      }",{"title":775,"description":776,"authors":781,"heroImage":777,"date":783,"body":784,"category":785,"tags":786},[782],"Matthieu Fronton","2022-09-12","If you've ever worked with GitLab CI/CD you may have needed, at some point,\nto use a cache to share content between jobs. The decentralized nature of\nGitLab CI/CD is a strength that can confuse the understanding of even the\nbest of us when we want to connect wires all together. For instance, we need\nto know critical information such as the difference between artifacts and\ncache and where/how to place setups.\n\n\nThis visual guide will help with both challenges.\n\n\n## Cache vs. artifacts\n\n\nThe concepts _may_ seem to overlap because they are about sharing content\nbetween jobs, but they actually are fundamentally different:\n\n\n- If your job does not rely on the the previous one (i.e. can produce it by\nitself but if content already exists the job will run faster), then use\ncache.\n\n- If your job does rely on the output of the previous one (i.e. cannot\nproduce it by itself), then use artifacts and dependencies.\n\n\nHere is a simple sentence to remember if you struggle between choosing cache\nor artifact:\n\n> Cache is here to speed up your job but it may not exist, so don't rely on\nit.\n\n\nThis article will focus on **cache**.\n\n\n## Initial setup\n\n\nWe'll go with a simple representation of the GitLab CI/CD pipelining model\nand ignore (for now) that the jobs can be executed on any runners and hosts.\nIt will help get the basics.\n\n\nLet's say you have:\n\n- 1 project with 3 branches\n\n- 1 host running 2 docker runners\n\n\n![Initial\nsetup](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-1.png){:\n.shadow.center}\n\n\n## Local cache: Docker volume\n\n\nIf you want a [local\ncache](https://docs.gitlab.com/ee/ci/caching/index.html#where-the-caches-are-stored)\nbetween all your jobs running on the same runner, use the [cache\nstatement](https://docs.gitlab.com/ee/ci/yaml/#cache) in your\n`.gitlab-ci.yml`:\n\n\n```yaml\n\ndefault:\n  cache:\n    path:\n      - relative/path/to/folder/*.ext\n      - relative/path/to/another_folder/\n      - relative/path/to/file\n```\n\n\n![local / container / all branches / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-2.png){:\n.shadow.center}\n\n\nUsing the [predefined\nvariable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\n`CI_COMMIT_REF_NAME` as the [cache\nkey](https://docs.gitlab.com/ee/ci/yaml/index.html#cachekey), you can ensure\nthe cache is tied to a specific branch:\n\n\n```yaml\n\ndefault:\n  cache:\n    key: $CI_COMMIT_REF_NAME\n    path:\n      - relative/path/to/folder/*.ext\n      - relative/path/to/another_folder/\n      - relative/path/to/file\n```\n\n\n![local / container / one branch / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-3.png){:\n.shadow.center}\n\n\nUsing the [predefined\nvariable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\n`CI_JOB_NAME` as the [cache\nkey](https://docs.gitlab.com/ee/ci/yaml/index.html#cachekey), you can ensure\nthe cache is tied to a specific job:\n\n\n![local / container / all branch / one\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-4.png){:\n.shadow.center}\n\n\n## Local cache: Bind mount\n\n\nIf you don't want to use a volume for caching purposes (debugging purpose,\ncleanup disk space more easily, etc.), you can configure a [bind mount for\nDocker volumes](https://docs.docker.com/storage/bind-mounts/) while\nregistering the runner. With this setup, you do not need to set up the\n[cache statement](https://docs.gitlab.com/ee/ci/yaml/#cache) in your\n`.gitlab-ci.yml`:\n\n\n```yaml\n\n#!/bin/bash\n\n\ngitlab-runner register                             \\\n  --name=\"Bind-Mount Runner\"                       \\\n  --docker-volumes=\"/host/path:/container/path:rw\" \\\n...\n\n```\n\n\n![local / one runners / one host / all branch / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-5.png){:\n.shadow.center}\n\n\nIn fact, this setup even allows you to share a cache between jobs running on\nthe same host without requiring you to set up a distributed cache (which\nwe'll talk about later):\n\n\n```yaml\n\n#!/bin/bash\n\n\ngitlab-runner register                             \\\n  --name=\"Bind-Mount Runner X\"                     \\\n  --docker-volumes=\"/host/path:/container/path:rw\" \\\n...\n\n\ngitlab-runner register                                 \\\n  --name=\"Bind-Mount Runner Y\"                         \\\n  --docker-volumes=\"/host/path:/container/alt/path:rw\" \\\n...\n\n```\n\n\n![local / multiple runners / one host / all branch / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-6.png){:\n.shadow.center}\n\n\n## Distributed cache\n\n\nIf you want to have a [shared\ncache](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching)\nbetween all your jobs running on multiple runners and hosts, use the \u003Ca\nhref=\"https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscache-section\">[runner.cache]\u003Ca>\nsection in your `config.toml`:\n\n\n```yaml\n\n[[runners]]\n  name = \"Distributed-Cache Runner\"\n...\n  [runners.cache]\n    Type = \"s3\"\n    Path = \"bucket/path/prefix\"\n    Shared = true\n    [runners.cache.s3]\n      ServerAddress = \"s3.amazonaws.com\"\n      AccessKey = \"\u003Cchangeme>\"\n      SecretKey = \"\u003Cchangeme>\"\n      BucketName = \"foobar\"\n      BucketLocation = \"us-east-1\"\n```\n\n\n![remote / multiple runners / multiple hosts / all branch / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-7.png){:\n.shadow.center}\n\n\nUsing the predefined variable `CI_COMMIT_REF_NAME` as the cache key you can\nensure the cache is tied to a specific branch between multiple runners and\nhosts:\n\n\n![remote / multiple runners / multiple hosts / one branch / all\njobs](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-8.png){:\n.shadow.center}\n\n\n## Real-life setup\n\n\nThe above assumptions allowed you to harness your understanding of the\nconcepts and possibilities.\n\n\nIn real life, you'll face more complex wiring and we hope this article will\nhelp you as a visual cheatsheet along with the reference documentation.\n\n\nJust to give you a sneak peek, here is an exercise for you:\n\n\n- Set up a cache between all the jobs of a specific stage, running on any\nrunner and any hosts, but only between pipeline of the same branches:\n\n\n![Real-life test\nassignment](https://about.gitlab.com/images/blogimages/visual-guide-caching/vgc-9.png){:\n.shadow.center}\n\n\nHappy caching, folks!\n\n\n\n\nCover image by [Alina Grubnyak](https://unsplash.com/@alinnnaaaa) on\n[Unsplash](https://unsplash.com)\n\n{: .note}\n","engineering",[787,788,789,9],"CI","CD","DevOps",{"slug":791,"featured":6,"template":700},"a-visual-guide-to-gitlab-ci-caching","content:en-us:blog:a-visual-guide-to-gitlab-ci-caching.yml","A Visual Guide To Gitlab Ci Caching","en-us/blog/a-visual-guide-to-gitlab-ci-caching.yml","en-us/blog/a-visual-guide-to-gitlab-ci-caching",{"_path":797,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":798,"content":804,"config":811,"_id":813,"_type":13,"title":814,"_source":15,"_file":815,"_stem":816,"_extension":18},"/en-us/blog/accelerate-code-reviews-with-gitlab-duo-and-amazon-q",{"title":799,"description":800,"ogTitle":799,"ogDescription":800,"noIndex":6,"ogImage":801,"ogUrl":802,"ogSiteName":685,"ogType":686,"canonicalUrls":802,"schema":803},"Accelerate code reviews with GitLab Duo and Amazon Q","Use AI-powered agents to optimize code reviews by automatically analyzing merge requests and providing comprehensive feedback on bugs, readability, and coding standards.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750096976/Blog/Hero%20Images/Blog/Hero%20Images/Screenshot%202024-11-27%20at%204.55.28%E2%80%AFPM_4VVz6DgGBOvbGY8BUmd068_1750096975734.png","https://about.gitlab.com/blog/accelerate-code-reviews-with-gitlab-duo-and-amazon-q","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Accelerate code reviews with GitLab Duo and Amazon Q\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2025-06-02\",\n      }",{"title":799,"description":800,"authors":805,"heroImage":801,"date":806,"body":807,"category":718,"tags":808},[715],"2025-06-02","Code reviews are critical for catching bugs, improving code readability, and maintaining coding standards, but they can also be a major bottleneck in your workflow. When you're trying to ship features quickly, waiting for multiple team members to review your code can be frustrating. The back-and-forth discussions, the scheduling conflicts, and the time it takes to get everyone aligned can stretch what should be a simple review into days or even weeks.\n\nHere's where [GitLab Duo with Amazon Q](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/), our new offering that delivers agentic AI throughout the software development lifecycle for AWS customers, comes in to transform your review process. This intelligent, AI-powered solution can perform comprehensive code reviews for you in a fraction of the time it would take your human colleagues. By leveraging advanced agentic AI capabilities, GitLab Duo with Amazon Q streamlines your entire review workflow without sacrificing the quality and thoroughness you need. Think of it as having an always-available, highly skilled reviewer who can instantly analyze your code and provide actionable feedback.\n\n## How it works: Launching a code review\n\nSo how does GitLab Duo with Amazon Q actually work? Let's say you've just finished working on a feature and created a merge request with multiple code updates. Instead of pinging your teammates and waiting for their availability, you simply enter a quick command in the comment section: \"/q review\". That's it – just those two words trigger the AI to spring into action.\n\n![Triggering a code review using GitLab Duo with Amazon Q](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097002/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097002096.png)\n\nOnce you've entered the command, Amazon Q Service immediately begins analyzing your code changes. You'll see a confirmation that the review is underway, and within moments, the AI is examining every line of your updates, checking for potential issues across multiple dimensions.\nWhen the review completes, you receive comprehensive feedback that covers all the bases: bug detection, readability improvements, syntax errors, and adherence to your team's coding standards. The AI doesn't just point out problems, it provides context and suggestions for fixing them, making it easy for you to understand what needs attention and why.\n\nThe beauty of this agentic AI approach is that it handles the heavy lifting of code review while you focus on what matters most: building great software. You get the benefits of thorough code reviews — better bug detection, consistent coding standards, and improved code quality — without the time sink. Your deployment times shrink dramatically because you're no longer waiting in review queues, and your entire team becomes more productive.\n\n## Why use GitLab Duo with Amazon Q?\n\nGitLab Duo with Amazon Q transforms your development workflow in the following ways:\n- Lightning-fast code reviews that don't compromise on quality\n- Consistent application of coding standards across your entire codebase\n- Immediate feedback that helps you fix issues before they reach production\n- Reduced deployment times that let you ship features faster\n- More time for your team to focus on creative problem-solving instead of repetitive reviews\n\nReady to see this game-changing feature in action? Watch how GitLab Duo with Amazon Q can revolutionize your code review process:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/4gFIgyFc02Q?si=GXVz--AIrWiwzf-I\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n> To learn more about GitLab Duo with Amazon Q visit us at an upcoming [AWS Summit in a city near you](https://about.gitlab.com/events/aws-summits/) or [reach out to your GitLab representative](https://about.gitlab.com/partners/technology-partners/aws/#form).\n> \n> And make sure to join the GitLab 18 virtual launch event to learn about our agentic AI plans and more. [Register today!](https://about.gitlab.com/eighteen/)",[720,495,809,694,696,283,810,9],"code review","AWS",{"slug":812,"featured":90,"template":700},"accelerate-code-reviews-with-gitlab-duo-and-amazon-q","content:en-us:blog:accelerate-code-reviews-with-gitlab-duo-and-amazon-q.yml","Accelerate Code Reviews With Gitlab Duo And Amazon Q","en-us/blog/accelerate-code-reviews-with-gitlab-duo-and-amazon-q.yml","en-us/blog/accelerate-code-reviews-with-gitlab-duo-and-amazon-q",{"_path":818,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":819,"config":822,"content":824,"_id":832,"_type":13,"title":833,"_source":15,"_file":834,"_stem":835,"_extension":18},"/en-us/blog/accelerate-learning-with-gitlab-duo-agent-platform",{"title":820,"description":821},"Accelerate learning with GitLab Duo Agent Platform","Learn how agentic AI helped generate comprehensive gRPC documentation in minutes, not hours.",{"slug":823,"featured":90,"template":700},"accelerate-learning-with-gitlab-duo-agent-platform",{"title":820,"description":821,"authors":825,"heroImage":827,"date":828,"category":718,"tags":829,"body":831},[826],"Halil Coban","Blog/Hero%20Images/Workflow_1800x945.png","2025-07-07",[718,9,694,830],"security","At GitLab, we continue to expand our AI capabilities so I often find myself learning and working in new codebases. Whether I'm debugging issues, implementing new features, or onboarding to different projects, understanding system architecture quickly is crucial. But let's be honest — manually tracing through complex communication flows, especially gRPC connections, can eat up hours of productive development time.\n\nThis is exactly the type of tedious, yet necessary, work [GitLab Duo Agent Platform](https://about.gitlab.com/blog/gitlab-duo-agent-platform-what-is-next-for-intelligent-devsecops/) is designed to handle. Instead of replacing developers, it amplifies our capabilities by automating routine tasks so we can focus on creative problem solving and strategic technical work.\n\nLet me show you how I used [Duo Agent Platform](https://about.gitlab.com/gitlab-duo/agent-platform/) to generate comprehensive documentation for a Golang project's gRPC communication flow — and how it transformed hours of code analysis into a few minutes of guided interaction.\n\nYou can follow along with this video:\n\n\u003Cdiv style=\"padding:75% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1098569263?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write; encrypted-media; web-share\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"AI Agent Generates Complete gRPC Documentation in Minutes | GitLab Duo Agent Platform Demo\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n## The challenge: Understanding gRPC communication flows\n\nI was working with a project called \"Duo Workflow Executor\" that communicates with a gRPC server. Rather than spending my afternoon manually tracing through the codebase to understand the communication patterns, I decided to let Duo Agent Platform handle the heavy lifting.\n\nMy goal was simple: generate a clear diagram showing how the gRPC communication works, including what payloads are received, what actions are executed, and what responses are sent back.\n\nWorking in VS Code with the GitLab Workflow extension installed, I opened the project and crafted a specific prompt for Duo Agent Platform:\n\n\"Can you prepare a mermaid diagram that shows the gRPC connection between duo-workflow-service and this project. It should show what this project receives in gRPC payload, and what actions it executes based on the payload, and what it sends back. Study internal/services/runner/runner.go, especially the Run method, and write the mermaid output to a grpc.md file.\"\n\nDuo Agent Platform didn't just blindly execute my request — it began intelligently gathering context to create a comprehensive execution plan. The platform automatically:\n\n* Searched through relevant Go files in the project  \n* Read the specific file I mentioned (runner.go)  \n* Identified additional files that would provide necessary context  \n* Analyzed the codebase structure to understand the gRPC implementation\n\nThis contextual awareness is what sets agentic AI tools with great context apart from simple code generation tools. Instead of working in isolation, Duo Agent Platform understands the relationships between different components of your project.\n\n## Collaborative refinement\n\nRather than making assumptions, Duo Agent Platform is designed to keep a human in the loop at critical moments in task execution to deliver better results. In this scenario, the platform paused to ask clarifying questions about the level of detail I wanted. \n\n**Agent:** \"Should I include error handling details in the diagram?\" \n\n**Me:** \"Don't focus on error handling. Include all actions. Focus on gRPC payload.\"\n\nThis back-and-forth refinement is crucial. The agent isn't just executing commands — it's working with me to ensure the deliverable meets my specific requirements.\n\n## Execution and results\n\nWith the clarified requirements, Duo Agent Platform finalized its execution plan and requested confirmation before proceeding. I could have modified the approach at this stage, but the plan was perfectly aligned with what I needed.\n\nThe agent then executed each task in sequence, providing real-time progress updates. Within minutes, it had:\n\n* Analyzed all relevant files across the project  \n* Created comprehensive documentation with a detailed mermaid diagram  \n* Saved everything to the requested grpc.md file  \n* Included valuable context about outgoing payloads and connection management\n\nThe resulting documentation wasn't just a basic diagram — it was thorough, accurate, and immediately useful for understanding the system architecture.\n\n## Reviewing the output\n\nHere's the actual mermaid diagram that Duo Agent Platform created, showing the complete gRPC communication flow:\n\n```mermaid\nsequenceDiagram\nparticipant DWS as duo-workflow-service\nparticipant DWE as duo-workflow-executor\nparticipant Runner as ExecutionRunner\nparticipant ActionExecutor as Action Executor\nparticipant FileSystem as File System/Commands\n\n\nNote over DWS, DWE: gRPC Bidirectional Streaming (ExecuteWorkflow)\n\n%% Connection Establishment\nDWE->>DWS: gRPC Connection with Headers\nNote over DWE, DWS: Headers: authorization (Bearer token)\u003Cbr/>x-gitlab-authentication-type: oidc\u003Cbr/>x-gitlab-realm, x-gitlab-global-user-id\u003Cbr/>x-gitlab-oauth-token, x-gitlab-base-url\u003Cbr/>x-gitlab-instance-id, x-request-id\u003Cbr/>x-gitlab-namespace-id, x-gitlab-project-id\n\n\n%% Workflow Start Request\nDWE->>DWS: ClientEvent{StartWorkflowRequest}\nNote over DWE, DWS: StartWorkflowRequest:\u003Cbr/>- ClientVersion\u003Cbr/>- WorkflowDefinition\u003Cbr/>- Goal\u003Cbr/>- WorkflowID\u003Cbr/>- WorkflowMetadata\u003Cbr/>- ClientCapabilities[]\n\n\n%% Action Processing Loop\nloop Action Processing\n    DWS->>DWE: Action Message\n    Note over DWS, DWE: Action Types:\u003Cbr/>- Action_RunCommand {program, flags[], arguments[]}\u003Cbr/>- Action_RunGitCommand {command, arguments[], repositoryUrl}\u003Cbr/>- Action_RunReadFile {filepath}\u003Cbr/>- Action_RunWriteFile {filepath, contents}\u003Cbr/>- Action_RunEditFile {filepath, oldString, newString}\u003Cbr/>- Action_RunHTTPRequest {method, path, body}\u003Cbr/>- Action_ListDirectory {directory}\u003Cbr/>- Action_FindFiles {namePattern}\u003Cbr/>- Action_Grep {searchDirectory, pattern, caseInsensitive}\u003Cbr/>- Action_NewCheckpoint {}\u003Cbr/>- Action_RunMCPTool {}\n\n\n    DWE->>Runner: Receive Action\n    Runner->>Runner: processWorkflowActions()\n    Runner->>ActionExecutor: executeAction(ctx, action)\n    \n    alt Action_RunCommand\n        ActionExecutor->>FileSystem: Execute Shell Command\n        Note over ActionExecutor, FileSystem: Executes: program + flags + arguments\u003Cbr/>in basePath directory\n        FileSystem-->>ActionExecutor: Command Output + Exit Code\n    \n    else Action_RunReadFile\n        ActionExecutor->>FileSystem: Read File\n        Note over ActionExecutor, FileSystem: Check gitignore rules\u003Cbr/>Read file contents\n        FileSystem-->>ActionExecutor: File Contents\n    \n    else Action_RunWriteFile\n        ActionExecutor->>FileSystem: Write File\n        Note over ActionExecutor, FileSystem: Check gitignore rules\u003Cbr/>Create/overwrite file\n        FileSystem-->>ActionExecutor: Success/Error Message\n    \n    else Action_RunEditFile\n        ActionExecutor->>FileSystem: Edit File\n        Note over ActionExecutor, FileSystem: Read → Replace oldString with newString → Write\u003Cbr/>Check gitignore rules\n        FileSystem-->>ActionExecutor: Edit Result Message\n    \n    else Action_RunGitCommand\n        ActionExecutor->>FileSystem: Execute Git Command \n        Note over ActionExecutor, FileSystem: Git operations with authentication\u003Cbr/>Uses provided git config\n        FileSystem-->>ActionExecutor: Git Command Output\n    \n    else Action_RunHTTPRequest\n        ActionExecutor->>DWS: HTTP Request to GitLab API\n        Note over ActionExecutor, DWS: Method: GET/POST/PUT/DELETE\u003Cbr/>Path: API endpoint\u003Cbr/>Body: Request payload\u003Cbr/>Headers: Authorization\n        DWS-->>ActionExecutor: HTTP Response\n    \n    else Action_ListDirectory\n        ActionExecutor->>FileSystem: List Directory Contents\n        Note over ActionExecutor, FileSystem: Respect gitignore rules\n        FileSystem-->>ActionExecutor: Directory Listing\n    \n    else Action_FindFiles\n        ActionExecutor->>FileSystem: Find Files by Pattern\n        Note over ActionExecutor, FileSystem: Recursive search with name pattern\u003Cbr/>Respect gitignore rules\n        FileSystem-->>ActionExecutor: File Paths List\n    \n    else Action_Grep\n        ActionExecutor->>FileSystem: Search Text Pattern\n        Note over ActionExecutor, FileSystem: Recursive text search\u003Cbr/>Case sensitive/insensitive option\n        FileSystem-->>ActionExecutor: Search Results\n    \n    else Action_NewCheckpoint/Action_RunMCPTool\n        ActionExecutor->>ActionExecutor: No-op Action\n        Note over ActionExecutor: Returns empty success result\n    end\n\n\n    ActionExecutor-->>Runner: Action Result (string)\n    \n    alt Result Size Check\n        Runner->>Runner: Check if result > 4MB\n        Note over Runner: If result exceeds MaxMessageSize (4MB)\u003Cbr/>Replace with error message about size limit\n    end\n\n\n    Runner->>DWE: ActionResponse\n    DWE->>DWS: ClientEvent{ActionResponse}\n    Note over DWE, DWS: ActionResponse:\u003Cbr/>- RequestID (matches Action.RequestID)\u003Cbr/>- Response (execution result string)\nend\n\n\n%% Workflow Completion\nDWE->>DWS: CloseSend()\nNote over DWE, DWS: Signal end of workflow execution\n\n\n%% Analytics and Cleanup\nRunner->>Runner: Send Analytics Event (Finish)\nDWE->>DWE: Token Revocation (if enabled)\nDWE->>DWS: Close gRPC Connection\n```\n\n\nThis diagram reveals several important architectural insights that would have taken considerable time to extract manually:\n\n* **Bidirectional communication:** The workflow executor both initiates requests and responds to service actions. \n* **Rich payload structure:** Each action type has specific parameters and expected responses.  \n* **Multiple integration points:** The executor interacts with local filesystem, Git repositories, and GitLab APIs.  \n* **Comprehensive action set:** Nine different action types handle everything from file operations to HTTP requests.  \n* **Proper lifecycle management:** Clear connection establishment and teardown patterns.\n\nWhat impressed me most was how the agent automatically included the detailed payload structures for each action type. This level of detail transforms the diagram from a high-level overview into actionable documentation that other developers can immediately use.\n\n## Looking ahead\n\nThis demonstration represents just one use case for GitLab Duo Agent Platform. The same contextual understanding and collaborative approach that made documentation generation seamless can be applied to:\n\n* **Code reviews:** Agents can analyze merge requests with full project context  \n* **Testing:** Generate comprehensive test suites based on actual usage patterns  \n* **Debugging:** Trace issues across multiple services and components  \n* **Security scanning:** Identify vulnerabilities with understanding of your specific architecture  \n* **CI/CD optimization:** Improve pipeline performance based on historical data\n\nGitLab Duo Agent Platform will enter public beta soon so [join the wait list today](https://about.gitlab.com/gitlab-duo/agent-platform/).\n\nStay tuned to the [GitLab Blog](https://about.gitlab.com/blog/) and social channels for additional updates. GitLab Duo Agent Platform is evolving rapidly with specialized agents, custom workflows, and community-driven extensions on the roadmap.\n\n## Learn more\n\n- [Agentic AI guides and resources](https://about.gitlab.com/blog/agentic-ai-guides-and-resources/)\n- [GitLab Duo Agent Platform: What’s next for intelligent DevSecOps](https://about.gitlab.com/blog/gitlab-duo-agent-platform-what-is-next-for-intelligent-devsecops/)\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n- [From vibe coding to agentic AI: A roadmap for technical leaders](https://about.gitlab.com/the-source/ai/from-vibe-coding-to-agentic-ai-a-roadmap-for-technical-leaders/)","content:en-us:blog:accelerate-learning-with-gitlab-duo-agent-platform.yml","Accelerate Learning With Gitlab Duo Agent Platform","en-us/blog/accelerate-learning-with-gitlab-duo-agent-platform.yml","en-us/blog/accelerate-learning-with-gitlab-duo-agent-platform",{"_path":837,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":838,"content":844,"config":852,"_id":854,"_type":13,"title":855,"_source":15,"_file":856,"_stem":857,"_extension":18},"/en-us/blog/actioning-security-vulnerabilities-in-gitlab-premium",{"title":839,"description":840,"ogTitle":839,"ogDescription":840,"noIndex":6,"ogImage":841,"ogUrl":842,"ogSiteName":685,"ogType":686,"canonicalUrls":842,"schema":843},"How to action security vulnerabilities in GitLab Premium","Learn step-by-step how to process detected vulnerabilities and spawn merge request approval rules from critical vulnerabilities.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099637/Blog/Hero%20Images/Blog/Hero%20Images/security-pipelines_security-pipelines.jpg_1750099637178.jpg","https://about.gitlab.com/blog/actioning-security-vulnerabilities-in-gitlab-premium","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to action security vulnerabilities in GitLab Premium\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sam Morris\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2023-03-13\",\n      }",{"title":839,"description":840,"authors":845,"heroImage":841,"date":848,"body":849,"category":830,"tags":850},[846,847],"Sam Morris","Noah Ing","2023-03-13","GitLab Premium features several security scanners you can leverage to detect\nvulnerabilities. However, when you incorporate the scanners into your\nproject pipelines and the scanning job succeeds, you'll want feedback on\nwhether you are introducing vulnerabilities into the codebase. This tutorial\nprovides a mechanism to require a merge request approval if a scanner\navailable on GitLab Premium finds a critical vulnerability.\n\n\n*While this tutorial shows how to add some process around actioning\nvulnerabilities, we have more robust, governed, and user-friendly\nfunctionality available in GitLab Ultimate called a [Scan Result\nPolicy](https://docs.gitlab.com/ee/user/application_security/policies/scan-result-policies.html).\nThe solution outlined here does not seek to replace that functionality, but\nrather augment the scan results available in GitLab Premium. If you are an\nUltimate user or if you want to compare the two experiences, then you should\ncheck out [this video\nintroduction](https://www.youtube.com/watch?v=w5I9gcUgr9U&ab_channel=GitLabUnfiltered)\ninstead.*\n\n\nLearn how to do the following:\n\n\n1. Set up a .gitlab-ci.yml\n\n2. Add in a vulnerability processing script\n\n3. Require approval if vulnerabilities are found \n\n\n### Prerequisites\n\n\n- A project with GitLab Premium\n\n- A gitlab-ci.yml\n\n- A project access token\n\n- Basic knowledge of Python\n\n- 5 minutes (or less)\n\n\n## Setup the gitlab-ci.yml \n\n\nThis is how the GitLab CI pipeline of our test project looks visually. Below\nwe will break down the individual stages.\n\n\nAdd the following to your .gitlab-ci.yml:\n\n\n```yaml\n\nsecret_detection:\n  artifacts:\n    paths:\n      - gl-secret-detection-report.json\n\nprocess_secret_detection:\n   image: python:3.7-alpine3.9\n   stage: process_vulns\n   needs:\n    - job: secret_detection\n      artifacts: true\n   before_script:\n      pip install python-gitlab\n   script:\n     - python3 process_vulns.py gl-secret-detection-report.json $PROJECT_ACCESS_TOKEN $CI_PROJECT_ID $CI_COMMIT_SHA\n```\n\n\nA breakdown of what is going on above:\n\n- gl-secret-detection-report.json needs to be overriden so it’s being stored\nas an artifact in the secret_detection job.\n\n- The process_secret_detection job is dependent on secret_detection's\nartifact so we have added a needs keyword requiring successful completion of\nthe secret_detection job.\n\n- pip installs the python-gitlab dependency so that the process_vulns.py can\nleverage GitLab API calls.\n\n- The process_vulns.py is taking in four arguments:\n   - gl-secret-detection-report.json is the JSON report produced from the secret_detection scanner. If you would like to take in another report this will need to be modified.\n   - $PROJECT_ACCESS_TOKEN needs to be added; review the instructions on creating a project access token in the next step.\n   - $CI_PROJECT_ID and $CI_COMMIT_SHA are both GitLab CI environment variables that will automatically be inferred.\n\n### Create a project access token\n\n\nTo create a project access token:\n\n1. On the top bar, select Main menu > Projects and find your project.\n\n2. On the left sidebar, select Settings > Access Tokens.\n\n3. Enter a name. The token name is visible to any user with permissions to\nview the project.\n\n4. Optional. Enter an expiry date for the token. The token expires on that\ndate at midnight UTC. An instance-wide maximum lifetime setting can limit\nthe maximum allowable lifetime in self-managed instances.\n\n5. Select a role for the token.\n\n6. Select the desired scopes.\n\n7. Select Create project access token.\n\n8. Add this newly created project access token to your CI/CD variables in\nyour project settings!\n\n\n## Add in the vulnerability processing script\n\n\n[The process_vulns.py script can be found\nhere.]((https://gitlab.com/gl-demo-premium-smorris/secure-premium-app/-/blob/main/process_vulns.py)\nCopy that file into your project.\n\n\nThe goal of this script is to require approval from an author (or group of\nauthors) if a critical vulnerability is found.\n\n\n**Note:** You will need to [change the user ID in the\nprocess_vulns.py](https://gitlab.com/gl-demo-premium-smorris/secure-premium-app/-/blame/main/process_vulns.py#L40)\nto match the user ID of your designated Approver at your organization.\n\n\nThe following is a breakdown of what the script is doing:\n\n\n- JSON security reports are loaded in, if there any vulnerabilities they are\nparsed.\n\n- An authentication with GitLab is run using the project access token to\ninteract with the project.\n\n- If vulnerabilities are not found, then it will print to the GitLab CI\nLogs: “No vulnerabilities are found.”\n\n- If a critical vulnerability is found, then it will require an approval.\n\n\nRun the pipeline and voila! Your pipeline now requires approvers if a\ncritical vulnerability is found!\n\n\n### Demo\n\n\nWatch a video demonstration of how to action security vulnerabilities in\nGitLab Premium, presented by Sam Morris:\n\n\n\u003Ciframe width=\"560\" height=\"315\"\nsrc=\"https://www.youtube.com/embed/Cld36OZrLFo\" title=\"YouTube video player\"\nframeborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write;\nencrypted-media; gyroscope; picture-in-picture; web-share\"\nallowfullscreen>\u003C/iframe>\n\n\n#### Caveats\n\n- This is mimicking a Scan Result Policy; it is not a replacement.\n\n- This currently only requires approval for a critical vulnerability, and\neach new rule would have to be added to the script.\n\n- This script lives within the same location as your project, so there is no\nrestriction on who can modify the script, breaking separation of duties at\nscale.\n\n- Approval rules are not removed once the vulnerability is fixed.\n\n- Approvers' IDs need to be hardcoded and maintained in the script file.\n\n- Since there is no vulnerability record generated, you cannot track the\nvulnerabilities over time in your application.\n\n- Vulnerabilities are not fed into a report or security dashboard, so this\nonly reports merge request vulnerabilities.\n\n\n## References\n\n- [Create a project access\ntoken](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#create-a-project-access-token)\n\n- [Setting up CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/#define-a-cicd-variable-in-the-ui)\n\n- [Secure Premium app\nproject](https://gitlab.com/gl-demo-premium-smorris/secure-premium-app/-/blob/main/process_vulns.py)\n\n\n## Related posts\n\n- [GitLab's commitment to enhanced application security in the modern DevOps\nworld](/blog/security-gitlab-15/)\n\n- [How to become more productive with Gitlab\nCI](/blog/how-to-become-more-productive-with-gitlab-ci/)\n\n- [GitLab CI DRY\nDevelopment](/blog/keeping-your-development-dry/)\n\n\n_Cover image by [Christopher\nBurns](https://unsplash.com/@christopher__burns?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non [Unsplash](https://www.unsplash.com)._\n",[830,851,787,788,9],"DevSecOps",{"slug":853,"featured":6,"template":700},"actioning-security-vulnerabilities-in-gitlab-premium","content:en-us:blog:actioning-security-vulnerabilities-in-gitlab-premium.yml","Actioning Security Vulnerabilities In Gitlab Premium","en-us/blog/actioning-security-vulnerabilities-in-gitlab-premium.yml","en-us/blog/actioning-security-vulnerabilities-in-gitlab-premium",{"_path":859,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":860,"content":866,"config":873,"_id":875,"_type":13,"title":876,"_source":15,"_file":877,"_stem":878,"_extension":18},"/en-us/blog/agentic-ai-guides-and-resources",{"title":861,"description":862,"ogTitle":861,"ogDescription":862,"noIndex":6,"ogImage":863,"ogUrl":864,"ogSiteName":685,"ogType":686,"canonicalUrls":864,"schema":865},"Agentic AI guides and resources","Learn everything you need to know about agentic AI, including what it is, how it works, why it levels up your DevSecOps environment, and best practices for implementation.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749658912/Blog/Hero%20Images/blog-image-template-1800x945__20_.png","https://about.gitlab.com/blog/agentic-ai-guides-and-resources","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Agentic AI guides and resources\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2025-05-07\",\n      }",{"title":861,"description":862,"authors":867,"heroImage":863,"date":869,"body":870,"category":718,"tags":871,"updatedDate":872},[868],"GitLab","2025-05-07","## Defining agentic AI\n\nAgentic AI is a type of artificial intelligence that leverages advanced language models and natural language processing to take independent action. Unlike traditional generative AI tools that require constant human direction, these systems can understand requests, make decisions, and execute multi-step plans to achieve goals. They tackle complex tasks by breaking them into manageable steps and employ adaptive learning to modify their approach when facing challenges.\n\n[Learn more about agentic AI](https://about.gitlab.com/topics/agentic-ai/)\n\n## Agentic AI insights\n- [GitLab Duo Agent Platform Public Beta: Next-gen AI orchestration and more](https://about.gitlab.com/blog/gitlab-duo-agent-platform-public-beta/) — Introducing the DevSecOps orchestration platform designed to unlock asynchronous collaboration between developers and AI agents.\n- [GitLab Duo Agent Platform: What's next for intelligent DevSecOps](https://about.gitlab.com/blog/gitlab-duo-agent-platform-what-is-next-for-intelligent-devsecops/) — GitLab Duo Agent Platform, a DevSecOps orchestration platform for humans and AI agents, leverages agentic AI for collaboration across the software development lifecycle.\n- [From vibe coding to agentic AI: A roadmap for technical leaders](https://about.gitlab.com/the-source/ai/from-vibe-coding-to-agentic-ai-a-roadmap-for-technical-leaders/) — Discover how to implement vibe coding and agentic AI in your development process to increase productivity while maintaining code quality and security.\n- [Emerging agentic AI trends reshaping software development](https://about.gitlab.com/the-source/ai/emerging-agentic-ai-trends-reshaping-software-development/) — Discover how agentic AI transforms development from isolated coding to intelligent workflows that enhance productivity while maintaining security.\n- [Agentic AI: Unlocking developer potential at scale](https://about.gitlab.com/the-source/ai/agentic-ai-unlocking-developer-potential-at-scale/) — Explore how agentic AI is transforming software development, moving beyond code completion to create AI partners that proactively tackle complex tasks.\n- [Agentic AI, self-hosted models, and more: AI trends for 2025](https://about.gitlab.com/the-source/ai/ai-trends-for-2025-agentic-ai-self-hosted-models-and-more/) — Discover key trends in AI for software development, from on-premises model deployments to intelligent, adaptive AI agents.\n- [How agentic AI unlocks platform engineering potential](https://about.gitlab.com/the-source/ai/how-agentic-ai-unlocks-platform-engineering-potential/) — Explore how agentic AI elevates platform engineering by automating complex workflows and scaling standardization.\n\n## The agentic AI ecosystem\n- [AI-driven code analysis: The new frontier in code security](https://about.gitlab.com/topics/agentic-ai/ai-code-analysis/) \n- [DevOps automation & AI agents](https://about.gitlab.com/topics/agentic-ai/devops-automation-ai-agents/)\n- [AI-augmented software development: Agentic AI for DevOps](https://about.gitlab.com/topics/agentic-ai/ai-augmented-software-development/)\n\n## Best practices for implementing agentic AI\n\n- [Implementing effective guardrails for AI agents](https://about.gitlab.com/the-source/ai/implementing-effective-guardrails-for-ai-agents/) — Discover essential security guardrails for AI agents in DevSecOps, from compliance controls and infrastructure protection to user access management.\n\n## GitLab's agentic AI offerings\n\n### GitLab Duo with Amazon Q\n\n- [GitLab Duo with Amazon Q: Agentic AI optimized for AWS generally available](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/) — The comprehensive AI-powered DevSecOps platform combined with the deepest set of cloud computing capabilities speeds dev cycles, increases automation, and improves code quality.\n- [DevSecOps + Agentic AI: Now on GitLab Self-Managed Ultimate on AWS](https://about.gitlab.com/blog/devsecops-agentic-ai-now-on-gitlab-self-managed-ultimate-on-aws/) — Start using AI-powered, DevSecOps-enhanced agents in your AWS GitLab Self-Managed Ultimate instance. Enjoy the benefits of GitLab Duo and Amazon Q in your organization.\n- [GitLab Duo with Amazon Q partner page](https://about.gitlab.com/partners/technology-partners/aws/)\n\nWatch GitLab Duo with Amazon Q in action:\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1075753390?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write; encrypted-media\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Technical Demo: GitLab Duo with Amazon Q\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n#### Guided tour\nClick on the image to start a tour of GitLab Duo with Amazon Q:\n\n[![GitLab Duo with Amazon Q interactive tour](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749673568/Blog/Content%20Images/Screenshot_2025-05-07_at_7.24.45_AM.png)](https://gitlab.navattic.com/duo-with-q)\n\n#### GitLab Duo with Amazon Q tutorials\n- [Enhance application quality with AI-powered test generation](https://about.gitlab.com/blog/enhance-application-quality-with-ai-powered-test-generation/) — Learn how GitLab Duo with Amazon Q improves the QA process by automatically generating comprehensive unit tests.\n- [GitLab Duo + Amazon Q: Transform ideas into code in minutes](https://about.gitlab.com/blog/gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes/) — The new GitLab Duo with Amazon Q integration analyzes your issue descriptions and automatically generates complete working code solutions, accelerating development workflows.\n- [Accelerate code reviews with GitLab Duo and Amazon Q](https://about.gitlab.com/blog/accelerate-code-reviews-with-gitlab-duo-and-amazon-q/) — Use AI-powered agents to optimize code reviews by automatically analyzing merge requests and providing comprehensive feedback on bugs, readability, and coding standards.\n- [Speed up code reviews: Let AI handle the feedback implementation](https://about.gitlab.com/blog/speed-up-code-reviews-let-ai-handle-the-feedback-implementation/) — Discover how GitLab Duo with Amazon Q automates the implementation of code review feedback through AI, transforming a time-consuming manual process into a streamlined workflow.\n\n### GitLab Duo Agent Platform\n- [GitLab Duo Chat gets agentic AI makeover](https://about.gitlab.com/blog/gitlab-duo-chat-gets-agentic-ai-makeover/) — Our new Duo Chat experience, currently an experimental release, helps developers onboard to projects, understand assignments, implement changes, and more.\nWatch GitLab Duo Agent Platform in action:\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1095679084?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write; encrypted-media; web-share\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Agent Platform Demo Clip\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n#### GitLab Agent Platform (and Duo Workflow) tutorials and use cases\n- [Accelerate learning with GitLab Duo Agent Platform](https://about.gitlab.com/blog/accelerate-learning-with-gitlab-duo-agent-platform/) — Learn how agentic AI helped generate comprehensive gRPC documentation in minutes, not hours.\n- [Fast and secure AI agent deployment to Google Cloud with GitLab](https://about.gitlab.com/blog/fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab/)\n\n- [Refactoring JavaScript to TypeScript with GitLab Duo Workflow](https://about.gitlab.com/blog/refactoring-javascript-to-typescript-with-gitlab-duo-workflow/)\n\n- [Automating tedious coding tasks with GitLab Duo Workflow](https://about.gitlab.com/blog/automate-tedious-coding-tasks-with-gitlab-duo-workflow/) — See how agentic AI can reduce time spent on repetitive tasks, freeing you up to focus on developing innovative solutions and shipping the next big thing.\n\n- [Use GitLab Duo Workflow to improve application quality assurance](https://about.gitlab.com/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance/) — Learn step-by-step how to add unit tests to a Java application using agentic AI (includes a video tutorial).\n\n- [Solving complex challenges with GitLab Duo Workflow](https://about.gitlab.com/blog/solving-complex-challenges-with-gitlab-duo-workflow/) — Learn how a member of the GitLab Customer Success Management team uses agentic AI for real-world problem-solving, including addressing Helm chart limits in the package registry.\n\n- [Get started with GitLab Duo Agentic Chat in the web UI](https://about.gitlab.com/blog/get-started-with-gitlab-duo-agentic-chat-in-the-web-ui/) — Learn about our new GitLab Duo AI feature that automates tasks by breaking down complex problems and executing operations across multiple sources.\n\n- [Custom rules in GitLab Duo Agentic Chat for greater developer efficiency](https://about.gitlab.com/blog/custom-rules-duo-agentic-chat-deep-dive/) — Discover how AI can understand your codebase, follow your conventions, and generate production-ready code with minimal review cycles.\n\n## Learn more with GitLab University\n\n- [Get Started with GitLab Duo coursework](https://university.gitlab.com/pages/ai)\n- [GitLab Duo Enterprise Learning Path](https://university.gitlab.com/learning-paths/gitlab-duo-enterprise-learning-path)\n\n## More AI resources\n\n- [2024 Global DevSecOps Survey: Navigating AI maturity in DevSecOps](https://about.gitlab.com/developer-survey/2024/ai/)\n- [The Role of AI in DevOps](https://about.gitlab.com/topics/devops/the-role-of-ai-in-devops/)\n- [The latest AI/ML articles from GitLab](https://about.gitlab.com/blog/categories/ai-ml/)\n- [GitLab Duo](https://about.gitlab.com/gitlab-duo/)",[720,495,9],"2025-06-10",{"slug":874,"featured":90,"template":700},"agentic-ai-guides-and-resources","content:en-us:blog:agentic-ai-guides-and-resources.yml","Agentic Ai Guides And Resources","en-us/blog/agentic-ai-guides-and-resources.yml","en-us/blog/agentic-ai-guides-and-resources",{"_path":880,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":881,"content":887,"config":894,"_id":896,"_type":13,"title":897,"_source":15,"_file":898,"_stem":899,"_extension":18},"/en-us/blog/amazon-linux-2-support-and-distro-specific-packages",{"title":882,"description":883,"ogTitle":882,"ogDescription":883,"noIndex":6,"ogImage":884,"ogUrl":885,"ogSiteName":685,"ogType":686,"canonicalUrls":885,"schema":886},"Amazon Linux 2 support and distro-specific packages for GitLab","Learn how to do early testing as well as how to peg your automation to the EL 7 packages until you are able to properly integrate the changes into your automation.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682299/Blog/Hero%20Images/gitlab-blog-banner.png","https://about.gitlab.com/blog/amazon-linux-2-support-and-distro-specific-packages","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Amazon Linux 2 support and distro-specific packages for GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Darwin Sanoy\"}],\n        \"datePublished\": \"2022-05-02\",\n      }",{"title":882,"description":883,"authors":888,"heroImage":884,"date":889,"body":890,"category":891,"tags":892},[691],"2022-05-02","GitLab’s Distribution Engineering team has been hard at work getting Amazon\nLinux 2 distro-specific packages ready in preparation for GitLab’s official\nsupport of Amazon Linux 2. Starting with Version 15.0 of GitLab, Amazon\nLinux 2 is a supported distro and packages are available for both x86 and\nGraviton ARM architectures.\n\n\n## What is Amazon Linux 2?\n\n\nAmazon Linux 2 is the next-generation Amazon Linux operating system that\nprovides a modern application environment with the most recent enhancements\nfrom the Linux community alongside long-term support. Amazon Linux 2 is\naccessible as a virtual machine image for on-premises development and\ntesting. This lets you easily develop, test, and certify your applications\nright from your local development environment. \n\n\nAccording to the AWS FAQ page for Amazon Linux 2, the primary elements of\nthis latest version of the operating system include:\n\n\n1. A Linux kernel tuned for performance on Amazon EC2.\n\n\n2. A set of core packages including systemd, GCC 7.3, Glibc 2.26, Binutils\n2.29.1 that receive Long Term Support (LTS) from\n[AWS](/blog/deploy-aws/).\n\n\n3. An extras channel for rapidly evolving technologies that are likely to be\nupdated frequently and outside the Long Term Support (LTS) model.\n\n\nAmazon Linux 2 has a support lifespan through June 20, 2024, to allow enough\ntime for users to migrate to Amazon Linux 2022.\n\n\n\n## Safely moving forward to Amazon Linux 2 packages from EL7\n\n\nWhile Amazon Linux 2 has not been officially supported before 15.0, as a\nconvenience to customers who wanted to use yum and RPM packages to install\nthe EL7 packages, GitLab configured a workaround in our packaging services\nto direct Amazon Linux 2 yum requests to the EL7 packages. If you’ve been\nusing GitLab’s yum repo registration script, you many not know that you were\nusing EL7 packages and not distro-specific packages.\n\n\nThis workaround will be deprecated and requests from Amazon Linux 2 will get\nthe distro-specific packages with the release of GitLab 15.3.0 on August 22,\n2022.\n\n\nAs a convenience for those of you who have automation that depends directly\non this workaround, we wanted to provide you with information on how to do\nearly testing as well as how to peg your automation to the EL 7 packages\nuntil you are able to properly integrate the changes into your automation.\n\n\nGitLab documentation demonstrates how to call our managed yum repository\nsetup scripts by downloading the latest copy and running it directly in [the\ninstructions for installing\ninstances](https://about.gitlab.com/install/#centos-7) and [the instructions\nfor installing\nrunners](https://docs.gitlab.com/runner/install/linux-repository.html).\n\n\nAny organization using GitLab’s EL 7 packages for Amazon Linux 2 will want\nto test with - and update to - the distro-specific packages as soon as\npossible as GitLab will only be testing Amazon Linux 2 with the Amazon Linux\n2 specific packages going forward.\n\n\nWe also understand that the timing of the testing and migration to these\npackages must be done in a coordinated cutover so that the package type does\nnot change in your existing stacks without you having made any changes. This\ncan be more important if a GitLab stack has undergone platform qualification\nfor compliance purposes.\n\n\nAmazon Linux 2 specific packages are only available for GitLab 14.9.0 and\nlater. If your automation depends directly on GitLab’s repo configuration\nscript and it is still pegged to a GitLab version prior to 14.9.0 when this\nchange becomes GA, then action must be taken to prevent breaking that\nautomation. We have devised an idempotent two-line script solution that you\ncan put in place now to prevent disruption if you are still on a pre-14.9.0\nversion at the time the new behavior of `script.rpm.sh` becomes GA on August\n22, 2022 with the release of GitLab 15.3.0.\n\n\nGitLab rake-based backup and restore will continue to work seamlessly across\nthe distro-specific package changes if you have to restore to your Amazon\nLinux 2 built stack from an EL7 backup. If you are using third-party backup,\nyou may wish to trigger a new backup immediately after transitioning to the\nnew distro packages to avoid the scenario altogether.\n\n\n## Amazon Linux 2 packages for building GitLab instances before 15.3.0\n\n\nThe following code inserts two lines of code between those originally\noutlined in [the instructions for installing using RPM\npackages](/install/#centos-7). The first one (starts with `sed`) splices in\nthe Amazon Linux 2 yum repo endpoint edits into the repository configuration\nfile created by script.rpm.sh. The second one (starts with `if yum`) cleans\nthe yum cache if the package was already installed so that the new location\nwill be used.\n\n\n> Sudo note: If you are using these commands interactively under the default\nSSH or SSM session manager user, then using `sudo su` before running this\ncode is necessary. If you are using these commands in Infrastructure as Code\n(e.g. CloudFormation userdata scripts), then sudo may cause ‘command not\nfound’ errors when the user running automation is already root equivalent.\nBe mindful about using interactively tested commands directly in your\nautomation.\n\n\n```bash\n\n#Existing packaging script from https://about.gitlab.com/install/#centos-7\n\ncurl\nhttps://packages.gitlab.com/install/repositories/gitlab/gitlab-ee/script.rpm.sh\n| sudo bash\n\n\n#Patch to preview and/or peg Amazon Linux 2 specific packages\n\nsed -i \"s/\\/el\\/7/\\/amazon\\/2/g\" /etc/yum.repos.d/gitlab_gitlab*.repo\n\n\n#Reset the cache if the package was previously installed (not needed for\ninstalls onto a clean machine)\n\nif yum list installed gitlab-ee; then yum clean all ; yum makecache; fi\n\n\n#Existing install command (remove \"-y\" to validate package and arch mapping\nbefore install)\n\nyum install gitlab-ee -y\n\n```\n\n\n> Notice in this output that the **Version** ends in `.amazon2`. In this\ncase the **Arch** is `aarch64` - indicating 64-bit Graviton ARM.\n\n\n![Resolved GitLab\nDependencies](https://about.gitlab.com/images/blogimages/2022-04-amazon-linux-2/gl-instance-dependencies-resolved.png)\n\n\n### Moving to Amazon Linux 2 packages early for a seamless post-GA\ntransition\n\n\nWhen the script.rpm.sh script is cut over to always point Amazon Linux 2 to\nthe new distro-specific packages, the sed command will no longer be\nnecessary. However, sed is also idempotent and will not make edits if the\nsearch text is not found. This means you can use the sed command to switch\nover early, but not have to worry about a breaking change when the\n`script.rpm.sh` is updated.\n\n\n### Pegging EL 7 and/or a GitLab version prior to 14.9.0 for a seamless\npost-GA transition\n\n\nIf your automation is pegged to an earlier version of GitLab, you will need\nto keep using EL7 packages, and, in fact, after the cutover you would need\nto implement the opposite command (which is also idempotent to be\nimplemented now).\n\n\n```bash\n\n#Patch to peg GitLab Version to EL 7 Packages (only does something after GA\nof gitlab repo script)\n\nsed -i \"s/\\/amazon\\/2/\\/el\\/7/g\" /etc/yum.repos.d/gitlab_gitlab*.repo\n\n\n#Reset the cache if the package was previously installed (not needed for\ninstalls onto a clean machine)\n\nif yum list installed gitlab-ee; then yum clean all ; yum makecache; fi\n\n```\n\n\nJust like the sed command for taking distro-specific packages early, this\ncommand can be implemented immediately with no bad effects - which will\nseamlessly keeping your automation pegged to the EL 7 packages when\n`script.rpm.sh` is updated.\n\n\n## Amazon Linux 2 package for building GitLab Runners before 15.3.0\n\n\nThe following code inserts two lines of code between those originally\n[outlined in the\ninstructions](https://docs.gitlab.com/runner/install/linux-repository.html).\nThe first one (starts with `sed`) splices in the Amazon Linux 2 yum repo\nendpoint edits into the repository configuration file created by\nscript.rpm.sh. The second one (starts with `if yum`) cleans the yum cache if\nthe package was already installed so that the new location will be used.\n\n\n> Sudo note: If you are using these commands interactively under the default\nSSH or SSM session manager user, then using `sudo su` before running this\ncode is necessary. If you are using these commands in Infrastructure as Code\n(e.g. CloudFormation userdata scripts), then sudo may cause ‘command not\nfound’ errors when the user running automation is already root equivalent.\nBe mindful about using interactively tested commands directly in your\nautomation.\n\n\n```bash\n\n#Existing packaging script from\nhttps://docs.gitlab.com/runner/install/linux-repository.html\n\ncurl -L\n\"https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh\"\n| sudo bash\n\n\n#Patch to test or peg Amazon Linux 2 specific packages\n\nsed -i \"s/\\/el\\/7/\\/amazon\\/2/g\" /etc/yum.repos.d/runner_gitlab*.repo\n\n\n#Reset the cache if the package was previously installed (not needed for\ninstalls onto a clean machine)\n\nif yum list installed gitlab-runner; then yum clean all ; yum makecache; fi\n\n\n#Existing install command (remove \"-y\" to validate package and arch mapping\nbefore install)\n\nyum install gitlab-runner -y\n\n```\n\n\n> Notice in this output that **Version** is not distro-specific. In this\ncase the **Arch** is `aarch64` - indicating 64-bit Graviton ARM.\n\n\n![Resolved GitLab Runner\nDependencies](https://about.gitlab.com/images/blogimages/2022-04-amazon-linux-2/gl-runner-dependencies-resolved.png)\n\n\n## Pegging to EL 7 and/or a GitLab Runner version prior to 14.9.1 for a\nseamless post-GA transition\n\n\nThe underlying package for EL 7 and Amazon Linux 2 is literally a copy of\nthe same package. However, the Amazon Linux 2 endpoint for Runner RPM\npackages have only been uploaded from GitLab Runner 14.9.1 and later, so if\nyou have runners that need to be on an earlier version, you would need to\nstay pointed at EL 7 for those packages to continue to resolve as available.\nThe following code shows how to do that for GitLab Runner.\n\n\n```bash\n\n#Patch to peg GitLab Version to EL 7 Packages (only does something after GA\nof gitlab repo script)\n\nsed -i \"s/\\/amazon\\/2/\\/el\\/7/g\" /etc/yum.repos.d/runner_gitlab*.repo\n\n\n#Reset the cache if the package was previously installed (not needed for\ninstalls onto a clean machine)\n\nif yum list installed gitlab-runner; then yum clean all ; yum makecache; fi\n\n```\n\n\n## Need-to-know takeaways\n\n\n- Amazon Linux 2 is a supported distro for GitLab instances and runner as of\nthe release of version 15.0 on May 22, 2022.\n\n- Amazon Linux 2 packages are available for x86 and ARM for GitLab Version\n14.9.0 and higher. (Prior to 14.9.0 the EL7 packages must be used as they\nhave a long version history).\n\n- This is the first availability of ARM RPM packages of GitLab for Amazon\nLinux 2.\n\n- In 15.3 (August 22, 2022), the script.rpm.sh will automatically start\ndirecting to the Amazon Linux 2 packages where it had previously directed\nAmazon Linux 2 yum requests to the EL7 packages.\n\n- It is common to have taken a dependency directly on the latest version of\nthis GitLab script in other automation.\n\n- Before the GA cutover date of August 22, 2022 (15.3.0 GitLab Release), for\nthese scripts, you have the opportunity to pre-test these packages and\ndetermine whether they create any issues with your automation or GitLab\nconfiguration.\n\n- You can also peg to the Amazon Linux 2 packages early or peg to the EL7\npackages in advance if you find problems that you need more time to resolve.\nBoth of these pegging types are idempotent, meaning the code changes do not\ndo anything that causes problems after the change over happens.\n\n- Existing Amazon Linux 2 installations that were installed using the EL7\npackages can use a regular yum upgrade command to start using the new Amazon\nLinux 2 packages. This operation may also be an upgrade of the product\nversion at the same time. For existing installations you will need to patch\nthe yum repo files as explained in this article in order to upgrade directly\nto Amazon Linux 2 from EL7 using packages. \n\n\n> **Note**\n\n> This blog post and linked pages contain information related to upcoming\nproducts, features, and functionality. It is important to note that the\ninformation presented is for informational purposes only. Please do not rely\non this information for purchasing or planning purposes. As with all\nprojects, the items mentioned in this blog post and linked pages are subject\nto change or delay. The development, release, and timing of any products,\nfeatures, or functionality remain at the sole discretion of GitLab Inc.\n\n\n![AWS Partner\nLogo](https://about.gitlab.com/images/blogimages/2022-04-amazon-linux-2/awsgravitonready.png){:\n.right}\n","news",[893,787,788,9,810],"releases",{"slug":895,"featured":6,"template":700},"amazon-linux-2-support-and-distro-specific-packages","content:en-us:blog:amazon-linux-2-support-and-distro-specific-packages.yml","Amazon Linux 2 Support And Distro Specific Packages","en-us/blog/amazon-linux-2-support-and-distro-specific-packages.yml","en-us/blog/amazon-linux-2-support-and-distro-specific-packages",{"_path":901,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":902,"content":908,"config":914,"_id":916,"_type":13,"title":917,"_source":15,"_file":918,"_stem":919,"_extension":18},"/en-us/blog/android-cicd-with-gitlab",{"title":903,"description":904,"ogTitle":903,"ogDescription":904,"noIndex":6,"ogImage":905,"ogUrl":906,"ogSiteName":685,"ogType":686,"canonicalUrls":906,"schema":907},"Tutorial: Android CI/CD with GitLab","Learn how to create an automated Android CI/CD pipeline using GitLab and fastlane.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749669316/Blog/Hero%20Images/angela-compagnone-4Iyg6cNU7sI-unsplash.jpg","https://about.gitlab.com/blog/android-cicd-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Android CI/CD with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Darby Frey\"}],\n        \"datePublished\": \"2023-06-13\",\n      }",{"title":903,"description":904,"authors":909,"heroImage":905,"date":911,"body":912,"category":785,"tags":913},[910],"Darby Frey","2023-06-13","\n\nMention the word keystore and all Android developers in a 5km radius will suddenly have a small feeling of panic. Attempting to automate a [CI/CD](https://docs.gitlab.com/ee/ci/) pipeline to deploy an app can be frustrating, and configuring Google Play access and code signing is at the heart of the problem.\n\nBut fear not! GitLab Mobile DevOps is here to make this process easier and faster, and I am here to guide you.\n\n[GitLab Mobile DevOps](https://docs.gitlab.com/ee/ci/mobile_devops.html) is a collection of features built right into GitLab to solve the biggest challenges mobile teams face in establishing a DevOps practice.\n\nIn this blog post, I’ll demonstrate how to set up an automated CI/CD pipeline using GitLab and [fastlane](https://fastlane.tools/).\n\n## Prerequisites \nTo get started, there are a few prerequisites you’ll need:\n\n* A Google Play developer account - [https://play.google.com/console](https://play.google.com/console)\n* Ruby and Android Studio installed on your local machine [https://docs.fastlane.tools/getting-started/android/setup/](https://docs.fastlane.tools/getting-started/android/setup/)\n\n> Try your hand at the [iOS CI/CD for GitLab tutorial](https://about.gitlab.com/blog/ios-cicd-with-gitlab/)\n\n## Reference project\nFor this tutorial, we’ll use the Android demo project for reference: [https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo).\n\n## Install fastlane\nIf you haven’t done so yet, the first step will be to install fastlane. Do this by creating a file in the root of your project called `Gemfile`. Give it the following contents:\n\n```ruby\nsource \"https://rubygems.org\"\n\ngem \"fastlane\"\n```\n\nThen, from the terminal in your project, run:\n\n```\nbundle install.\n```\n\nThis command will install fastlane, and all of its related dependencies.\n\n## Initialize fastlane\nNow that fastlane is installed, we can set it up for our project. Run the following command from the terminal in your project. You’ll be asked to enter your package name, so enter that. When prompted for the JSON secret file, you can skip that for now, and you can answer \"no\" to the questions about metadata management.\n\n```\nbundle exec fastlane init\n```\n\n![Initialize fastlane](https://about.gitlab.com/images/blogimages/2023-06-13-android-cicd-with-gitlab/fastlane-init.png)\n\nRunning this command will create a new folder in your project called `fastlane`. This folder will contain two files `Appfile` and `Fastfile`.\n\nThe Appfile contains the configuration information for the app, and the Fastfile has some sample code that we will replace later. See the fastlane docs for more information about the configuration details in the Appfile: [https://docs.fastlane.tools/advanced/Appfile/](https://docs.fastlane.tools/advanced/Appfile/).\n\n## Code signing\nNext are the steps for code signing.\n\n### Create a keystore\nThe next step is to create a keystore and properties files for code signing. Run the following command to generate a keystore in the project root called `release-keystore.jks`:\n\n```\nkeytool -genkey -v -keystore release-keystore.jks -storepass password -alias release -keypass password -keyalg RSA -keysize 2048 -validity 10000\n```\n\n![Create a keystore](https://about.gitlab.com/images/blogimages/2023-06-13-android-cicd-with-gitlab/keytool-genkey.png)\n\nMore information is available in the [keytool docs](https://download.java.net/java/early_access/loom/docs/specs/man/keytool.html).\n\nThe next step is to create a properties file to be used by [Gradle](https://gradle.org/_). Create a file in the project root called `release-keystore.properties`, with the following contents:\n\n```\nstoreFile=../release-keystore.jks\nkeyAlias=release\nkeyPassword=password\nstorePassword=password\n```\n\nAlso, be sure to add both files to your `.gitignore` file so they aren't committed to version control.\n\n### Configure Gradle\nNext, configure Gradle to use the newly created keystore. In the `app/build.gradle` file, add the following:\n\n**1.** Right after the plugins section, add:\n\n```\ndef keystoreProperties = new Properties()\ndef keystorePropertiesFile = rootProject.file('release-keystore.properties')\nif (keystorePropertiesFile.exists()) {\n    keystoreProperties.load(new FileInputStream(keystorePropertiesFile))\n}\n```\n\n**2.** Before Build Types, add:\n\n```\nsigningConfigs {\n    release {\n   \t keyAlias keystoreProperties['keyAlias']\n   \t keyPassword keystoreProperties['keyPassword']\n   \t storeFile keystoreProperties['storeFile'] ? file(keystoreProperties['storeFile']) : null\n   \t storePassword keystoreProperties['storePassword']\n    }\n}\n```\n\n**3.** Lastly, add the signingConfig to the release build type:\n\n```\nsigningConfig signingConfigs.release\n```\n\n## Upload keystore to GitLab secure files\nNext, upload your keystore files to GitLab so they can be used in CI/CD jobs. \n\n1. On the top bar, select **Menu > Projects** and find your project.\n1. On the left sidebar, select **Settings > CI/CD**.\n1. In the Secure Files section, select **Expand**.\n1. Select **Upload File**.\n1. Find the file to upload, select **Open**, and the file upload begins immediately. The file shows up in the list when the upload is complete.\n\nDo this for both the `release-keystore.jks` file and the `release-keystore.properties` file.\n\n![Upload Secure File](https://about.gitlab.com/images/blogimages/2023-06-13-android-cicd-with-gitlab/upload-secure-file.png)\n\n![List Secure Files](https://about.gitlab.com/images/blogimages/2023-06-13-android-cicd-with-gitlab/list-secure-files.png)\n\n## Create a CI/CD pipeline\n\nWith the configuration in place, now copy the contents of the .gitlab-ci.yml and fastlane/Fastfile below to the project.\n\nThis [.gitlab-ci.yml](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo/-/blob/main/.gitlab-ci.yml) has all the configuration needed to run the test, build, and beta jobs.\nThe [fastlane/Fastfile](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo/-/blob/main/fastlane/Fastfile) is an example that can be customized to specific project settings.\n\nNote: This fastlane configuration uses plugins. See the [docs](https://docs.fastlane.tools/plugins/using-plugins/) for instructions on how to configure your project for fastlane plugins.\n\n## Create an app in the Google Play Console\nNext, generate a build of your app locally and upload it to seed a new app entry in the Google Play Console. Run the following command locally:\n\n```\nbundle exec fastlane build\n```\n\nThis command will create a signed build of the app at\n\n```\nbuild/outputs/bundle/release/app-release.aab\n```\n\nWith the signed build ready to go, log in to the [Google Play Console](https://play.google.com/console) and create a new app and seed it with the initial build.\n\n## Configure Google Play integration\nThe last thing to set up is the Google Play integration in GitLab. To do so, first, create a Google service account.\n\n### Create a Google service account\nFollow the [instructions](https://docs.fastlane.tools/actions/supply/#setup) for setting up a service account in Google Cloud Platform and granting that account access to the project in Google Play.\n\n### Enable Google Play integration\nFollow the [instructions](https://docs.gitlab.com/ee/user/project/integrations/google_play.html) for configuring the Google Play integration by providing a package name and the JSON key file just generated for the service account.\n\nThis is a simplified CI/CD configuration that created three CI/CD jobs to run each of the lanes in fastlane on the GitLab Runners. The test and build jobs will run for all CI/CD pipelines, and the beta job will only be run on CI/CD pipelines on the main branch. The beta job is manually triggered, so you can control when the beta release is pushed to Google Play. \n\nWith these configurations in place, commit all of these changes and push them up to your project. The CI/CD pipeline will kick off, and you can see these jobs in action.\n",[789,108,9],{"slug":915,"featured":6,"template":700},"android-cicd-with-gitlab","content:en-us:blog:android-cicd-with-gitlab.yml","Android Cicd With Gitlab","en-us/blog/android-cicd-with-gitlab.yml","en-us/blog/android-cicd-with-gitlab",{"_path":921,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":922,"content":928,"config":936,"_id":938,"_type":13,"title":939,"_source":15,"_file":940,"_stem":941,"_extension":18},"/en-us/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd",{"title":923,"description":924,"ogTitle":923,"ogDescription":924,"noIndex":6,"ogImage":925,"ogUrl":926,"ogSiteName":685,"ogType":686,"canonicalUrls":926,"schema":927},"Container image provenance with Cosign in GitLab CI/CD","Use GitLab pipelines to automate building, signing, and annotating Docker images. This tutorial shares code to show you how. Try it out in your own organization.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098395/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2823%29_2w6waL76KROjhJHM2vXet6_1750098395162.png","https://about.gitlab.com/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Annotate container images with build provenance using Cosign in GitLab CI/CD\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"João Pereira\"},{\"@type\":\"Person\",\"name\":\"Tim Rizzi\"}],\n        \"datePublished\": \"2024-09-04\",\n      }",{"title":929,"description":924,"authors":930,"heroImage":925,"date":933,"body":934,"category":830,"tags":935},"Annotate container images with build provenance using Cosign in GitLab CI/CD",[931,932],"João Pereira","Tim Rizzi","2024-09-04","Container security has become a critical concern in software development. As\norganizations increasingly rely on containerized applications, ensuring the\nintegrity and traceability of container images is paramount. Enhancing the\nsecurity and traceability of your container images directly in your GitLab\nCI/CD pipeline can streamline your development process while significantly\nboosting your security posture.\n\n\nThis tutorial demonstrates setting up a GitLab pipeline to automate the\nprocess of building, signing, and annotating Docker images using Cosign and\nthe GitLab container registry. By integrating these practices, you'll secure\nyour images and ensure that each one is easily traceable, aligning with best\npractices in DevSecOps.\n\n\n## Background on container image security\n\n\nBefore we dive into the technical details, it's crucial to understand why\ncontainer image security is so important. In\n[microservices](https://about.gitlab.com/topics/microservices/) and\ncloud-native applications, containers have become the standard for packaging\nand deploying software. However, this widespread adoption has also made\ncontainers an attractive target for cyber attacks.\n\n\nContainer image security is a vital component of the broader [software\nsupply chain\nsecurity](https://about.gitlab.com/blog/the-ultimate-guide-to-software-supply-chain-security/)\nconcept. This encompasses all the tools, processes, and practices that\nensure your software's integrity, authenticity, and security from\ndevelopment to deployment. By securing your container images, you're\nprotecting your application and your entire software supply chain.\n\n\n## Introduction to Cosign\n\n\nEnter\n[Cosign](https://about.gitlab.com/blog/keyless-signing-with-cosign/),\na tool designed to address these security concerns. Cosign is part of the\nSigstore project, an open-source initiative aimed at improving the security\nof the software supply chain. Cosign allows developers to sign and verify\ncontainer images, ensuring their integrity and authenticity.\n\n\nKey benefits of Cosign include:\n\n\n- easy integration with existing CI/CD pipelines\n\n- support for various signing methods, including keyless signing\n\n- ability to attach and verify arbitrary metadata to container images\n\n\nBy incorporating Cosign into your GitLab CI/CD pipeline, you're taking a\nsignificant step towards robust\n[DevSecOps](https://about.gitlab.com/topics/devsecops/) practices.\n\n\n## Benefits of image signing and annotation\n\n\nImage signing serves as a seal of authenticity for your container images. It\nhelps prevent tampering and ensures that the image deployed in your\nproduction environment is precisely the one that passed through your secure\nbuild process.\n\n\nAnnotations, on the other hand, provide valuable metadata about the build\nprocess. This information is used for auditing and traceability. In a\nsecurity incident, having detailed provenance data can significantly speed\nup the investigation and remediation process.\n\n\n## GitLab CI/CD pipeline configuration\n\n\nLet's look at an example `.gitlab-ci.yml` file that outlines the process of\nbuilding, signing, and annotating a Docker image using Cosign:\n\n\n```yaml\n\nstages:\n  - build\n\nbuild_and_sign:\n  stage: build\n  image: docker:latest\n  services:\n    - docker:dind  # Enable Docker-in-Docker service to allow Docker commands inside the container\n  variables:\n    IMAGE_TAG: $CI_COMMIT_SHORT_SHA  # Use the commit short SHA as the image tag\n    IMAGE_URI: $CI_REGISTRY_IMAGE:$IMAGE_TAG  # Construct the full image URI with the registry, project path, and tag\n    COSIGN_YES: \"true\"  # Automatically confirm actions in Cosign without user interaction\n    FF_SCRIPT_SECTIONS: \"true\"  # Enables GitLab's CI script sections for better multi-line script output\n  id_tokens:\n    SIGSTORE_ID_TOKEN:\n      aud: sigstore  # Provide an OIDC token for keyless signing with Cosign\n  before_script:\n    - apk add --no-cache cosign jq  # Install Cosign (mandatory) and jq (optional)\n    - docker login -u \"gitlab-ci-token\" -p \"$CI_JOB_TOKEN\" \"$CI_REGISTRY\"  # Log in to the Docker registry using GitLab CI token\n  script:\n    # Build the Docker image using the specified tag and push it to the registry\n    - docker build --pull -t \"$IMAGE_URI\" .\n    - docker push \"$IMAGE_URI\"\n\n    # Retrieve the digest of the pushed image to use in the signing step\n    - IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' \"$IMAGE_URI\")\n\n    # Sign the image using Cosign with annotations that provide metadata about the build and tag annotation to allow verifying\n    # the tag->digest mapping (https://github.com/sigstore/cosign?tab=readme-ov-file#tag-signing)\n    - |\n      cosign sign \"$IMAGE_DIGEST\" \\\n        --annotations \"com.gitlab.ci.user.name=$GITLAB_USER_NAME\" \\\n        --annotations \"com.gitlab.ci.pipeline.id=$CI_PIPELINE_ID\" \\\n        --annotations \"com.gitlab.ci.pipeline.url=$CI_PIPELINE_URL\" \\\n        --annotations \"com.gitlab.ci.job.id=$CI_JOB_ID\" \\\n        --annotations \"com.gitlab.ci.job.url=$CI_JOB_URL\" \\\n        --annotations \"com.gitlab.ci.commit.sha=$CI_COMMIT_SHA\" \\\n        --annotations \"com.gitlab.ci.commit.ref.name=$CI_COMMIT_REF_NAME\" \\\n        --annotations \"com.gitlab.ci.project.path=$CI_PROJECT_PATH\" \\\n        --annotations \"org.opencontainers.image.source=$CI_PROJECT_URL\" \\\n        --annotations \"org.opencontainers.image.revision=$CI_COMMIT_SHA\" \\\n        --annotations \"tag=$IMAGE_TAG\"\n\n    # Verify the image signature using Cosign to ensure it matches the expected annotations and certificate identity\n    - |\n      cosign verify \\\n        --annotations \"tag=$IMAGE_TAG\" \\\n        --certificate-identity \"$CI_PROJECT_URL//.gitlab-ci.yml@refs/heads/$CI_COMMIT_REF_NAME\" \\\n        --certificate-oidc-issuer \"$CI_SERVER_URL\" \\\n        \"$IMAGE_URI\" | jq .  # Use jq to format the verification output for easier readability\n```\n\n\nLet's break down this pipeline configuration and understand each part in\ndetail.\n\n\n## Detailed explanation of the pipeline\n\n\n### 1. Setup and prerequisites\n\n\nThe pipeline starts by setting up the necessary environment:\n\n\n* It uses the `docker:latest` image and enables Docker-in-Docker service,\nallowing Docker commands to be run within the CI job.\n\n* It defines variables for the image tag and URI using GitLab CI/CD\npredefined variables.\n\n* It sets up an OIDC token for keyless signing with Cosign.\n\n* In the `before_script` section, it installs Cosign and jq (for JSON\nprocessing) and logs into the GitLab container registry.\n\n\n### 2. Building and pushing the image\n\n\nThe first step in the script is to build the Docker image and push it to the\nGitLab container registry:\n\n\n```yaml\n\n- docker build --pull -t \"$IMAGE_URI\" .\n\n- docker push \"$IMAGE_URI\"\n\n```\n\n\nThis creates the image using the current directory's Dockerfile and pushes\nit to the registry.\n\n\n### 3. Signing the image with Cosign\n\n\nAfter building and pushing the image, the pipeline signs it using Cosign:\n\n\n```yaml\n\n- IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}'\n\"$IMAGE_URI\")\n\n- |\n  cosign sign \"$IMAGE_DIGEST\" \\\n    --annotations \"com.gitlab.ci.user.name=$GITLAB_USER_NAME\" \\\n    --annotations \"com.gitlab.ci.pipeline.id=$CI_PIPELINE_ID\" \\\n    # ... (other annotations) ...\n    --annotations \"tag=$IMAGE_TAG\"\n```\n\n\nThis step first retrieves the image digest and then uses Cosign to sign the\nimage, adding several annotations.\n\n\n## Verifying the signature and annotations\n\n\nAfter signing the image, it's crucial to verify the signature and the\nannotations we've added. This verification step ensures that the provenance\ndata attached to the image is correct and hasn't been tampered with.\n\n\nIn our pipeline, we've included a verification step using the `cosign\nverify` command:\n\n\n```yaml\n\n- |\n  cosign verify \\\n    --annotations \"tag=$IMAGE_TAG\" \\\n    --certificate-identity \"$CI_PROJECT_URL//.gitlab-ci.yml@refs/heads/$CI_COMMIT_REF_NAME\" \\\n    --certificate-oidc-issuer \"$CI_SERVER_URL\" \\\n    \"$IMAGE_URI\" | jq .\n```\n\n\nThis command verifies the signature and checks the annotations. Its output\nwill show all the annotations we've added to the image during the signing\nprocess.\n\n\nHere's what you might see in your pipeline logs after running this command:\n\n\n![verifying the signature and checking\nannotations](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098404/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098404260.png)\n\n\nIn this output, you should see all the annotations we added earlier,\nincluding:\n\n\n* GitLab CI user name\n\n* Pipeline ID and URL\n\n* Job ID and URL\n\n* Commit SHA and reference name\n\n* Project path\n\n* Image source and revision\n\n\nBy verifying these annotations, you can ensure that the image's provenance\ndata is intact and matches what you expect based on your build process. This\nverification step is crucial for maintaining the integrity of your software\nsupply chain. It allows you to confirm that the image you're about to deploy\nhas gone through your secure build process and has yet to be modified since\nit was signed.\n\n\n## Summary\n\n\nBy integrating Cosign into your GitLab CI/CD pipeline, you've taken a\nsignificant step toward securing your software supply chain. This setup not\nonly automates securing and annotating your container images with build\nmetadata but also ensures a transparent and traceable build process.\n\n\nThe benefits of this approach are numerous:\n\n\n- enhanced security through image signing\n\n- improved traceability with detailed build provenance data\n\n- automated verification process\n\n- alignment with DevSecOps best practices\n\n\nAs container security continues to be a critical concern in the software\ndevelopment lifecycle, implementing these practices puts you ahead of\npotential security threats and demonstrates a commitment to software\nintegrity.\n\n\n## Try it in your organization\n\n\nNow that you've seen how to enhance your container security using Cosign in\nGitLab CI/CD, it's time to put this knowledge into practice:\n\n\n1. **Implement in your projects**: Adapt the provided `.gitlab-ci.yml` file\nto fit your specific needs.\n\n2. **Explore further**: Dive deeper into Cosign's capabilities. Consider\nexploring advanced features like policy enforcement or integration with\nvulnerability scanning tools.\n\n3. **Share your experience**: After implementing this in your projects,\nshare your experience with your team or the wider GitLab community. Your\ninsights could help others enhance their security practices.\n\n4. **Stay updated**: Container security is an evolving field. Check GitLab's\nblog and documentation for new features and best practices updates.\n\n5. **Contribute**: If you find ways to improve this process or encounter any\nissues, consider contributing to the GitLab or Cosign open-source projects.\n\n\nRemember, security is a journey, not a destination. By taking these steps,\nyou're securing your containers and contributing to a more secure software\necosystem for everyone.\n\n\nStart implementing these practices in your GitLab projects today, and take\nyour container security to the next level!\n\n\n> Get started today! Sign up for a [free trial of GitLab\nUltimate](https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/blog&glm_content=default-saas-trial)!\n\n\n## Read more\n\n\n- [Next-generation GitLab container registry goes\nGA](https://about.gitlab.com/blog/next-generation-gitlab-container-registry-goes-ga/)\n\n- [A beginner's guide to container\nsecurity](https://about.gitlab.com/topics/devsecops/beginners-guide-to-container-security/)\n\n- [DevSecOps basics, including\nsecurity](https://about.gitlab.com/topics/devsecops/)\n\n- [What is CI/CD?](https://about.gitlab.com/topics/ci-cd/)\n",[830,9,694,696],{"slug":937,"featured":6,"template":700},"annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd","content:en-us:blog:annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd.yml","Annotate Container Images With Build Provenance Using Cosign In Gitlab Ci Cd","en-us/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd.yml","en-us/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd",{"_path":943,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":944,"content":950,"config":957,"_id":959,"_type":13,"title":960,"_source":15,"_file":961,"_stem":962,"_extension":18},"/en-us/blog/automate-tedious-coding-tasks-with-gitlab-duo-workflow",{"title":945,"description":946,"ogTitle":945,"ogDescription":946,"noIndex":6,"ogImage":947,"ogUrl":948,"ogSiteName":685,"ogType":686,"canonicalUrls":948,"schema":949},"Automate tedious coding tasks with GitLab Duo Workflow","See how agentic AI can reduce time spent on repetitive tasks, freeing you up to focus on developing innovative solutions and shipping the next big thing.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662465/Blog/Hero%20Images/GitLab_Duo_Workflow_Unified_Data_Store__1_.png","https://about.gitlab.com/blog/automate-tedious-coding-tasks-with-gitlab-duo-workflow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Automate tedious coding tasks with GitLab Duo Workflow\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Jeff Park\"}],\n        \"datePublished\": \"2025-05-06\",\n      }",{"title":945,"description":946,"authors":951,"heroImage":947,"date":953,"body":954,"category":718,"tags":955},[952],"Jeff Park","2025-05-06","Working with large codebases often means spending significant time on repetitive tasks that, while necessary, don't really push your projects forward. The good news is that these tasks are great candidates to be completed with AI. Reducing the time spent on them will free you up to work on more important problems that you’re actually excited to tackle. With GitLab Duo Workflow, the time spent on these tasks will go from hours to minutes.\n\n[Duo Workflow](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/) is a powerful new agentic solution, currently in private beta, that lives in VS Code and is designed to help you complete complex development tasks. While many AI coding assistants are focused on helping developers write code, Duo Workflow understands your project structure, reads your files, and can make coordinated changes across your entire codebase.\n\nI created a demonstration that showcases how Duo Workflow can transform a tedious coding task into a streamlined process that saves you time and mental energy.\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1081627484?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write; encrypted-media\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Automate tedious coding tasks with GitLab Duo Workflow\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n## The challenge: Implementing a new lint rule\n\nIn this demo, we tackle a common scenario that many developers face: implementing a new lint rule and then updating multiple files across the codebase to comply with this rule. The specific issue involves validation errors occurring in several project files that need to be addressed consistently.\n\nRather than manually identifying and modifying each affected file one by one – a process that could take hours depending on the size of your codebase – we'll see how Duo Workflow can:\n\n1. Read and understand the details from an issue\n2. Analyze the project structure to identify affected files\n3. Create a comprehensive plan to implement the necessary changes\n4. Draft a new lint rule to prevent future occurrences\n5. Make consistent code changes across all relevant files\n6. Stage the changes for your review before any commits are made\n\nA simple prompt initiates the process:\n\n\"Read through issue #1 in this project and submit code changes to resolve it. Be sure to look at each tool file and make all appropriate changes.\"\n\nFrom there, Duo Workflow takes over – reading the issue, analyzing the files, creating a plan, and implementing the solution – all while keeping me informed of its progress and reasoning.\n\n## Why this matters for your development process\n\nWhat's particularly powerful about Duo Workflow is how it maintains awareness of this wider context throughout the entire process. It's not just making text replacements based on a large language model's training data – it's understanding the code, making intelligent decisions, and proposing a complete solution that you maintain full control over.\n\nThis approach offers several key benefits:\n\n* **Consistency in implementation:** Apply changes uniformly across files\n* **Time savings:** Focus your energy on creative problem-solving rather than repetitive tasks\n* **Reduced context switching:** Complete complex tasks without leaving your IDE\n* **Keeping a human in the loop:** Review all proposed modifications before committing\n\n## What's next\n\nGitLab Duo Workflow is part of our work to bring AI-powered capabilities to every stage of the software development lifecycle. While this demo focuses on code editing, the same approach can be applied to various development tasks:\n\n* Implementing new features based on issue descriptions\n* Fixing bugs with comprehensive test coverage\n* Refactoring legacy code to modern standards\n* Creating documentation from codebase analysis\n\nWe believe that by automating repetitive tasks, Duo Workflow helps you focus on what matters most – solving interesting problems and creating innovative solutions for your users.\n\n> GitLab Duo Workflow is currently available in private beta for GitLab Ultimate customers. [Sign up for the waitlist today!](https://about.gitlab.com/gitlab-duo/agent-platform/)\n\n## Learn more\n- [Use GitLab Duo Workflow to improve application quality assurance](https://about.gitlab.com/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance/)\n- [Solving complex challenges with GitLab Duo Workflow](https://about.gitlab.com/blog/solving-complex-challenges-with-gitlab-duo-workflow/)\n- [GitLab Duo Workflow: Enterprise visibility and control for agentic AI](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/)\n- [Emerging agentic AI trends reshaping software development](https://about.gitlab.com/the-source/ai/emerging-agentic-ai-trends-reshaping-software-development/)\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n",[720,495,696,694,9,956],"workflow",{"slug":958,"featured":90,"template":700},"automate-tedious-coding-tasks-with-gitlab-duo-workflow","content:en-us:blog:automate-tedious-coding-tasks-with-gitlab-duo-workflow.yml","Automate Tedious Coding Tasks With Gitlab Duo Workflow","en-us/blog/automate-tedious-coding-tasks-with-gitlab-duo-workflow.yml","en-us/blog/automate-tedious-coding-tasks-with-gitlab-duo-workflow",{"_path":964,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":965,"content":971,"config":977,"_id":979,"_type":13,"title":980,"_source":15,"_file":981,"_stem":982,"_extension":18},"/en-us/blog/automating-a-twitter-bot-using-gitlab-cicd",{"title":966,"description":967,"ogTitle":966,"ogDescription":967,"noIndex":6,"ogImage":968,"ogUrl":969,"ogSiteName":685,"ogType":686,"canonicalUrls":969,"schema":970},"How to automate a Twitter bot using GitLab CI/CD","This tutorial shows how to use the DevSecOps platform to create a set-and-forget Twitter bot.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749661856/Blog/Hero%20Images/ci-cd-demo.jpg","https://about.gitlab.com/blog/automating-a-twitter-bot-using-gitlab-cicd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to automate a Twitter bot using GitLab CI/CD\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Siddharth Mathur\"}],\n        \"datePublished\": \"2023-03-21\",\n      }",{"title":966,"description":967,"authors":972,"heroImage":968,"date":974,"body":975,"category":785,"tags":976},[973],"Siddharth Mathur","2023-03-21","\n\nGitLab's CI/CD pipelines are great for automating many things, like deployments to Google Kubernetes Engine and security scans. But did you know that you could use GitLab CI/CD pipelines to run a set-and-forget Twitter bot?\n\nMany organizations today are leveraging Twitter's API to [understand customer sentiment](https://developer.twitter.com/en/blog/success-stories/target), [track public health data](https://developer.twitter.com/en/blog/success-stories/penn), [perform financial analysis](https://developer.twitter.com/en/blog/success-stories/likefolio), and more. While these bots may be running on self-managed infrastrucuture or external services, you can simplify and consolidate your tooling by leveraging GitLab instead, making your bot easier to manage.\n\nWith GitLab's [Free tier](/pricing/), you can leverage 400 minutes of CI/CD run time per month to automatically analyze and post tweets. With GitLab [Premium](/pricing/premium) and [Ultimate](/pricing/ultimate), you'll get even more pipeline minutes to tweet more, run longer natural language processing analyses, or for other projects.\n\nSetting up a Twitter bot using GitLab is pretty simple. At the end of this blog, you'll have a project that looks like [this](https://gitlab.com/smathur/twitter-bot), and a Twitter account that automatically posts a simple tweet.\n\nTo get started, you'll need these prerequisites:\n- GitLab account (self-hosted with GitLab Runner(s) set up or on GitLab.com)\n- Twitter API credentials\n\nOnce you've generated your Twitter API credentials, we can start building out our bot in GitLab. In this blog, we'll leverage GitLab's Web IDE based on Visual Studio Code, but feel free to use a code editor of your choice.\n\n## Step 1: Write a Python script to post tweets\n\n![Navigate to the Web IDE](https://about.gitlab.com/images/blogimages/2023-03-10-automating-a-twitter-bot-using-gitlab-cicd/web-ide.png){: .shadow}\n\nCreate a new blank project in GitLab, and click the \"Web IDE\" button to start writing some code. In the Web IDE, create a new file called `run_bot.py`, and paste the following code (this is where you interact with the Twitter API):\n\n```python\nimport tweepy\nimport config\n\ndef set_up():\n\tauth = tweepy.OAuthHandler(config.consumer_key, config.consumer_secret_key)\n\tauth.set_access_token(config.access_token, config.access_token_secret)\n\tapi = tweepy.API(auth)\n\treturn api\n\ndef run(tweet):\n\tapi = set_up()\n\tapi.update_status(tweet)\n\nrun('It\\'s Tanuki time')\n```\n\n**Note:** If you're familiar with Python, you'll notice that we're importing a file called `config` with some variables that we're using. This `config` file doesn't exist yet, but we'll create it from within a GitLab pipeline, leveraging CI/CD variables to securely store and use our Twitter API credentials.\n\nCreate another file called `requirements.txt`, and paste the following line:\n\n```\ntweepy\n```\n\nChanges to files in the Web IDE will be automatically saved, so switch to the Git tab and commit your changes.\n\n## Step 2: Create a CI/CD pipeline to run your Python script\n\nNext, we'll create a CI/CD pipeline script to run our Twitter bot and post a tweet every time the pipeline is run. To do this, you can:\n1. Create a new file using the Web IDE called `.gitlab-ci.yml`, or\n2. Head to your GitLab project, and from the sidebar, click CI/CD > Editor.\n\nIf you see some default text in the pipeline configuration, delete everything to start with a clean slate.\n\nIn the pipeline YAML file, we'll first specify the Docker image we want to run the bot on:\n\n```yaml\nimage: python:latest\n```\n\n**Note:** Normally in a pipeline, we would define stages first and then write jobs that are each assigned to a specific stage. Since we're only running one job in this pipeline, we don't need to specify stages at the top of our pipeline configuration file.\n\nNext, we'll add a job called `run` that runs the Python script we created in the previous step. Inside this job, we'll add a `script` section to run some commands that will execute our Python script.\n\n```yaml\nrun:\n  script:\n    - echo \"consumer_key = '$CONSUMER_KEY'\" >> config.py\n    - echo \"consumer_secret_key = '$CONSUMER_SECRET'\" >> config.py\n    - echo \"access_token = '$ACCESS_TOKEN'\" >> config.py\n    - echo \"access_token_secret = '$ACCESS_SECRET'\" >> config.py\n    - pip install -r requirements.txt\n    - python3 run_bot.py\n```\n\nCommit your changes. The pipeline will automatically run, since you just made a change to the project files, but it will fail. This is because we are calling some CI/CD variables in the pipeline, which we haven't set yet. Let's go ahead and do that!\n\n## Step 3: Set CI/CD variables to store API tokens\n\nHead to your GitLab project and from the sidebar, go to Settings > CI/CD.\n\nExpand the \"Variables\" section and add the `ACCESS_SECRET`, `ACCESS_TOKEN`, `CONSUMER_KEY`, and `CONSUMER_SECRET` variables as shown below (these are your Twitter API credentials):\n\n![CI/CD variables](https://about.gitlab.com/images/blogimages/2023-03-10-automating-a-twitter-bot-using-gitlab-cicd/ci-cd-variables.png){: .shadow}\n\nNote that the secrets are masked to prevent them from showing up in job logs (check the \"Mask variable\" box when creating/editing the variable).\n\n## Step 4: Test and schedule your Twitter bot\n\nNow that we've got everything set up, all we need to do is run the bot. Go to CI/CD > Pipelines, and click \"Run pipeline\". Click \"Run pipeline\" again, and wait for the `run` job to finish. If you've set up your Twitter credentials correctly, you should see that the pipeline successfully ran, and a tweet was posted on your bot account!\n\n![Schedule a pipeline](https://about.gitlab.com/images/blogimages/2023-03-10-automating-a-twitter-bot-using-gitlab-cicd/schedule-pipeline.png){: .shadow}\n\nOnce you've verified that your pipeline runs successfully, schedule your pipeline to automatically run at a regular interval. Go to CI/CD > Schedules, and click \"New schedule\". Feel free to use one of the default provided intervals, or use cron to set a custom schedule. Specify a timezone, and ensure that the \"Active\" checkbox is checked. Finally, click \"Save pipeline schedule\". You'll see that your pipeline has been scheduled to run, and when it will run next.\n\nAnd that's it! You now have a fully-functional Twitter bot running on GitLab, using CI/CD pipelines to automatically post tweets. While this demo Twitter bot simply posts a specified text message, you can add your own logic to [generate sentences using AI](https://linguatools.org/language-apis/sentence-generating-api/), [perform sentiment analysis on other users' tweets](https://www.analyticsvidhya.com/blog/2021/06/twitter-sentiment-analysis-a-nlp-use-case-for-beginners/), and more. Running a Twitter bot is just one of the many ways you can leverage pipelines in GitLab, and you can also check out some other [interesting use cases](https://docs.gitlab.com/ee/ci/examples/).\n",[851,787,788,9],{"slug":978,"featured":6,"template":700},"automating-a-twitter-bot-using-gitlab-cicd","content:en-us:blog:automating-a-twitter-bot-using-gitlab-cicd.yml","Automating A Twitter Bot Using Gitlab Cicd","en-us/blog/automating-a-twitter-bot-using-gitlab-cicd.yml","en-us/blog/automating-a-twitter-bot-using-gitlab-cicd",{"_path":984,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":985,"content":991,"config":997,"_id":999,"_type":13,"title":1000,"_source":15,"_file":1001,"_stem":1002,"_extension":18},"/en-us/blog/automating-agile-workflows-with-the-gitlab-triage-gem",{"title":986,"description":987,"ogTitle":986,"ogDescription":987,"noIndex":6,"ogImage":988,"ogUrl":989,"ogSiteName":685,"ogType":686,"canonicalUrls":989,"schema":990},"Automating Agile workflows with the gitlab-triage gem","Learn how to automate repetitive tasks like triaging issues and merge requests to free up valuable developer time in our \"Getting Started with GitLab\" series.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659525/Blog/Hero%20Images/blog-getting-started-with-gitlab-banner-0497-option4-fy25.png","https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Automating Agile workflows with the gitlab-triage gem\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2025-03-13\",\n      }",{"title":986,"description":987,"authors":992,"heroImage":988,"date":993,"body":994,"category":694,"tags":995},[868],"2025-03-13","*Welcome to our \"Getting started with GitLab\" series, where we help\nnewcomers get familiar with the GitLab DevSecOps platform.*\n\n\nThis post dives into the\n[`gitlab-triage`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-triage)\ngem, a powerful tool that lets you create bots to automate your Agile\nworkflow. Say goodbye to manual tasks and hello to streamlined efficiency.\n\n\n## Why automate your workflow?\n\n\nEfficiency is key in software development. Automating repetitive tasks like\ntriaging issues and merge requests frees up valuable time for your team to\nfocus on what matters most: building amazing software.\n\n\nWith `gitlab-triage`, you can:\n\n\n* **Ensure consistency:** Apply labels and assign issues automatically based\non predefined rules.  \n\n* **Improve response times:** Get immediate feedback on new issues and merge\nrequests.  \n\n* **Reduce manual effort:** Eliminate the need for manual triage and\nupdates.  \n\n* **Boost productivity:** Free up your team to focus on coding and\ninnovation.\n\n\n## Introducing the `gitlab-triage` gem\n\n\nThe `gitlab-triage` gem is a Ruby library that allows you to create bots\nthat interact with your GitLab projects. These bots can automatically\nperform a wide range of actions, including:\n\n\n* **Labeling:** Automatically categorize issues and merge requests.  \n\n* **Commenting:** Provide updates, request information, or give feedback.  \n\n* **Assigning:** Assign issues and merge requests to the appropriate team\nmembers.  \n\n* **Closing:** Close stale or resolved issues and merge requests.  \n\n* **Creating:** Generate new issues based on specific events or\nconditions.  \n\n* **And much more!**\n\n\nCheck out the [`gitlab-triage` gem\nrepository](https://gitlab.com/gitlab-org/ruby/gems/gitlab-triage). \n\n\n## Setting up your triage bot\n\n\nLet's get your first triage bot up and running!\n\n\n1. Install the gem. (Note: The gem command is available with Ruby\nprogramming language installed.)\n\n\n```bash\n\ngem install gitlab-triage\n\n```\n\n\n2. Get your GitLab API token.\n\n\n* Go to your GitLab [profile\nsettings](https://gitlab.com/-/profile/preferences).  \n\n* Navigate to **Access Tokens**.  \n\n* Create a new token with the `api` scope.  \n\n* **Keep your token secure and set an expiration date for it based on when\nyou will be done with this walkthrough!**\n\n\n3. Define your triage policies.\n\n\nCreate a file named `.triage-policies.yml` in your project's root directory.\nThis file will contain the rules that govern your bot's behavior. Here's a\nsimple example:\n\n\n```yaml\n\n\n---\n\n- name: \"Apply 'WIP' label\"\n  condition:\n    draft: true\n  action:\n    labels:\n      - status::wip\n\n- name: \"Request more information on old issue\"\n  condition:\n   date:\n    attribute: updated_at\n    condition: older_than\n    interval_type: months\n    interval: 12\n  action:\n    comment: |\n      {{author}} This issue has been open for more than 12 months, is this still an issue?\n```\n\n\nThis configuration defines two policies:\n\n\n* The first policy applies the `status::wip` label to any issue that is in\ndraft.  \n\n* The second policy adds a comment to an issue that the issue has not been\nupdated in 12 months.\n\n\n4. Run your bot.\n\n\nYou can run your bot manually using the following command:\n\n\n```bash\n\ngitlab-triage -t \u003Cyour_api_token> -p \u003Cyour_project_id>\n\n```\n\n\nReplace `\u003Cyour_api_token>` with your GitLab API token and\n`\u003Cyour_project_id>` with the [ID of your GitLab\nproject](https://docs.gitlab.com/user/project/working_with_projects/#access-a-project-by-using-the-project-id).\nIf you would like to see the impact of actions before they are taken, you\ncan add the `-n` or `--dry-run` to test out the policies first.\n\n\n## Automating with GitLab CI/CD\n\n\nTo automate the execution of your triage bot, integrate it with [GitLab\nCI/CD](https://about.gitlab.com/blog/ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation/).\nHere's an example `.gitlab-ci.yml` configuration:\n\n\n```yaml\n\n\ntriage:\n  script:\n    - gem install gitlab-triage\n    - gitlab-triage -t $GITLAB_TOKEN -p $CI_PROJECT_ID\n  only:\n    - schedules\n```\n\n\nThis configuration defines a job named \"triage\" that installs the\n`gitlab-triage` gem and runs the bot using the `$GITLAB_TOKEN` (a predefined\n[CI/CD variable](https://docs.gitlab.com/ci/variables/)) and the\n`$CI_PROJECT_ID` variable. The `only: schedules` clause ensures that the job\nruns only on a schedule.\n\n\nTo create a\n[schedule](https://docs.gitlab.com/ee/ci/pipelines/schedules.html), go to\nyour project's **CI/CD** settings and navigate to **Schedules**. Create a\nnew schedule and define the frequency at which you want your bot to run\n(e.g., daily, hourly).\n\n\n## Advanced triage policies\n\n\n`gitlab-triage` offers a range of advanced features for creating more\ncomplex triage policies:\n\n\n* **Regular expressions:** Use regular expressions for more powerful pattern\nmatching.  \n\n* **Summary policies:** Consolidate related issues into a single summary\nissue.  \n\n* **Custom actions:** Define custom actions using [Ruby code\nblocks](https://gitlab.com/gitlab-org/ruby/gems/gitlab-triage#can-i-customize)\nto perform more complex operations using the GitLab API.\n\n\nHere are two advanced real-world examples from the triage bot used by the\nDeveloper Advocacy team at GitLab. You can view the full policies in [this\nfile](https://gitlab.com/gitlab-da/projects/devrel-bot/-/blob/master/.triage-policies.yml?ref_type=heads).\n\n\n```yaml\n\n- name: Issues where DA team member is an assignee outside DA-Meta project\ni.e. DevRel-Influenced\n  conditions:\n    assignee_member:\n      source: group\n      condition: member_of\n      source_id: 1008\n    state: opened\n    ruby: get_project_id != 18 \n    forbidden_labels:\n      - developer-advocacy\n  actions:   \n    labels:\n      - developer-advocacy\n      - DevRel-Influenced\n      - DA-Bot::Skip\n```\n\n\nThis example for issues across a group, excluding those in the project with\nthe ID of 18, have assignees who are members of the group with ID of 1008\nand do not have the label `developer-advocacy` on them. This policy helps\nthe Developer Advocacy team at GitLab to find issues members of the team are\nassigned to but are not in their team’s project. This helps the team\nidentify and keep track of contributions made outside of the team by adding\nthe teams’ labels.\n\n\n```\n\n- name: Missing Due Dates\n  conditions:\n    ruby: missing_due_date\n    state: opened\n    labels:\n      - developer-advocacy\n    forbidden_labels:\n      - DA-Due::N/A\n      - DA-Bot::Skip\n      - DA-Status::FYI\n      - DA-Status::OnHold\n      - CFP\n      - DA-Bot::Triage\n  actions:\n    labels:\n      - DA-Bot-Auto-Due-Date\n    comment: |\n      /due #{get_current_quarter_last_date}\n```\n\n\nThis second example checks for all issues with the `developer-advocacy`\nlabel, which do not include labels in the forbidden labels list and when\ntheir due dates have passed. It updates the due dates automatically by\ncommenting on the issue with a slash command and a date that is generated\nusing Ruby.\n\n\nThe Ruby scripts used in the policies are defined in a separate file as\nshown below. This feature allows you to be flexible in working with your\nfilters and actions. You can see functions are created for different Ruby\ncommands that we used in our policies. \n\n\n```\n\nrequire 'json'\n\nrequire 'date'\n\nrequire \"faraday\"\n\nrequire 'dotenv/load'\n\n\nmodule DATriagePlugin\n  def last_comment_at\n    conn = Faraday.new(\n      url: notes_url+\"?sort=desc&order_by=created_at&pagination=keyset&per_page=1\",\n      headers: {'PRIVATE-TOKEN' => ENV.fetch(\"PRIV_KEY\"), 'Content-Type' => 'application/json' }\n    )\n\n    response = conn.get()\n    if response.status == 200\n      jsonData = JSON.parse(response.body)\n      if jsonData.length > 0\n        Date.parse(jsonData[0]['created_at'])\n      else\n        Date.parse(resource[:created_at])\n      end\n    else\n      Date.parse(resource[:created_at])\n    end\n  end\n\n  def notes_url\n    resource[:_links][:notes]\n  end\n\n  def get_project_id\n    resource[:project_id]\n  end\n\n  def get_current_quarter_last_date()\n    yr = Time.now.year\n    case Time.now.month\n    when 2..4\n      lm = 4\n    when 5..7\n      lm = 7\n    when 8..10\n      lm = 10\n    when 11..12\n      lm = 1\n      yr = yr + 1\n    else\n      lm = 1    \n    end\n\n    return Date.new(yr, lm, -1) \n  end\n\n  def one_week_to_due_date\n    if(resource[:due_date] == nil)\n      false\n    else\n      days_to_due = (Date.parse(resource[:due_date]) - Date.today).to_i\n      if(days_to_due > 0 && days_to_due \u003C 7)\n        true\n      else\n        false\n      end\n    end\n  end\n\n  def due_date_past\n    if(resource[:due_date] == nil)\n      false\n    else\n      Date.today > Date.parse(resource[:due_date])\n    end\n  end\n\n  def missing_due_date\n    if(resource[:due_date] == nil)\n      true\n    else\n      false\n    end\n  end\n\nend\n\n\nGitlab::Triage::Resource::Context.include DATriagePlugin\n\n\n```\n\nThe triage bot is executed using the command:\n\n\n``` \n\n`gitlab-triage -r ./triage_bot/issue_triage_plugin.rb --debug --token\n$PRIV_KEY --source-id gitlab-com --source groups`  \n\n```\n\n\n- `-r`: Passes in a  file of requirements for the performing triage. In this\ncase we are passing in our Ruby functions.  \n\n- `--debug`: Prints debugging information as part of the output.  \n\n- `--token`: Is used to pass in a valid GitLab API token.  \n\n- `--source`: Specifies if the sources of the issues it will search is\nwithin a group or a project.  \n\n- `--source-id`: Takes in the ID of the selected source type – in this case,\na group.\n\n\nThe GitLab [triage-ops](https://gitlab.com/gitlab-org/quality/triage-ops)\nproject is another real-world example that is more complex and you can learn\nhow to build your own triage bot.\n\n\n## Best practices\n\n\n* **Start simple:** Begin with basic policies and gradually increase\ncomplexity as needed. \n\n* **Test thoroughly:** Test your policies in a staging environment before\ndeploying them to production.  \n\n* **Monitor regularly:** Monitor your bot's activity to ensure it's behaving\nas expected. \n\n* **Use descriptive names:** Give your policies clear and descriptive names\nfor easy maintenance. \n\n* **Be mindful of the scope of your filters:** You might be tempted to\nfilter issues across groups where thousands of issues exist. However, this\ncan slow down the triage and also make the process fail due to rate\nlimitations against the GitLab API.  \n\n* **Prioritize using labels for triages:** To avoid spamming other users,\nlabels are a good way to perform triages without cluttering comments and\nissues.\n\n\n## Take control of your workflow\n\n\nWith the `gitlab-triage` gem, you can automate your GitLab workflow and\nunlock new levels of efficiency. Start by creating simple triage bots and\ngradually explore the more advanced features. You'll be amazed at how much\ntime and effort you can save\\!\n\n\n> #### Want to take your learning to the next level? [Sign up for GitLab\nUniversity courses](https://university.gitlab.com/). Or you can get going\nright away with a [free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).\n\n\n## \"Getting started with GitLab\" series\n\nRead more articles in our \"Getting started with GitLab\" series:\n\n\n- [How to manage\nusers](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\n\n- [How to import your projects to\nGitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n\n- [Mastering project\nmanagement](https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management/)\n\n- [Understanding\nCI/CD](https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd/)\n\n- [Working with CI/CD\nvariables](https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables/)\n",[495,9,694,996,108],"agile",{"slug":998,"featured":6,"template":700},"automating-agile-workflows-with-the-gitlab-triage-gem","content:en-us:blog:automating-agile-workflows-with-the-gitlab-triage-gem.yml","Automating Agile Workflows With The Gitlab Triage Gem","en-us/blog/automating-agile-workflows-with-the-gitlab-triage-gem.yml","en-us/blog/automating-agile-workflows-with-the-gitlab-triage-gem",{"_path":1004,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1005,"content":1011,"config":1017,"_id":1019,"_type":13,"title":1020,"_source":15,"_file":1021,"_stem":1022,"_extension":18},"/en-us/blog/automating-container-image-migration-from-amazon-ecr-to-gitlab",{"title":1006,"description":1007,"ogTitle":1006,"ogDescription":1007,"noIndex":6,"ogImage":1008,"ogUrl":1009,"ogSiteName":685,"ogType":686,"canonicalUrls":1009,"schema":1010},"Automating container image migration from Amazon ECR to GitLab","When platform teams move their CI/CD to GitLab, migrating container images shouldn't be the bottleneck. Follow this step-by-step guide to automate the pipeline migration process.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663129/Blog/Hero%20Images/blog-image-template-1800x945__28_.png","https://about.gitlab.com/blog/automating-container-image-migration-from-amazon-ecr-to-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Automating container image migration from Amazon ECR to GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tim Rizzi\"}],\n        \"datePublished\": \"2025-02-13\",\n      }",{"title":1006,"description":1007,"authors":1012,"heroImage":1008,"date":1013,"body":1014,"category":785,"tags":1015},[932],"2025-02-13","\"We need to migrate hundreds of container images from Amazon Elastic\nContainer Registry (ECR) to GitLab. Can you help?\" This question kept coming\nup in conversations with platform engineers. They were modernizing their\nDevSecOps toolchain with GitLab but got stuck when faced with moving their\ncontainer images. While each image transfer is simple, the sheer volume made\nit daunting.\n\n\nOne platform engineer perfectly said, \"I know exactly what needs to be done\n– pull, retag, push. But I have 200 microservices, each with multiple tags.\nI can't justify spending weeks on this migration when I have critical\ninfrastructure work.\"\n\n\n## The challenge\n\n\nThat conversation sparked an idea. What if we could automate the entire\nprocess? When platform teams move their\n[CI/CD](https://about.gitlab.com/topics/ci-cd/) to GitLab, migrating\ncontainer images shouldn't be the bottleneck. The manual process is\nstraightforward but repetitive – pull each image, retag it, and push it to\nGitLab's Container Registry. Multiply this by dozens of repositories and\nmultiple tags per image, and you're looking at days or weeks of tedious\nwork.\n\n\n## The solution\n\n\nWe set out to create a GitLab pipeline that would automatically do all this\nheavy lifting. The goal was simple: Give platform engineers a tool they\ncould set up in minutes and let run overnight, waking up to find all their\nimages migrated successfully.\n\n\n### Setting up access\n\n\nFirst things first – security. We wanted to ensure teams could run this\nmigration with minimal AWS permissions. Here's the read-only identity and\naccess management (IAM) policy you'll need:\n\n\n```json\n\n{\n    \"Version\": \"2012-10-17\",\n    \"Statement\": [\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"ecr:GetAuthorizationToken\",\n                \"ecr:BatchCheckLayerAvailability\",\n                \"ecr:GetDownloadUrlForLayer\",\n                \"ecr:DescribeRepositories\",\n                \"ecr:ListImages\",\n                \"ecr:DescribeImages\",\n                \"ecr:BatchGetImage\"\n            ],\n            \"Resource\": \"*\"\n        }\n    ]\n}\n\n```\n\n\n### GitLab configuration\n\n\nWith security handled, the next step is setting up GitLab. We kept this\nminimal - you'll need to configure these variables in your CI/CD settings:\n\n\n```\n\nAWS_ACCOUNT_ID: Your AWS account number\n\nAWS_DEFAULT_REGION: Your ECR region\n\nAWS_ACCESS_KEY_ID: [Masked]\n\nAWS_SECRET_ACCESS_KEY: [Masked]\n\nBULK_MIGRATE: true\n\n```\n\n\n### The migration pipeline\n\n\nNow for the interesting part. We built the pipeline using Docker-in-Docker\nto handle all the image operations reliably:\n\n\n```yaml\n\nimage: docker:20.10\n\nservices:\n  - docker:20.10-dind\n\nbefore_script:\n  - apk add --no-cache aws-cli jq\n  - aws sts get-caller-identity\n  - aws ecr get-login-password | docker login --username AWS --password-stdin\n  - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}\n```\n\n\nThe pipeline works in three phases, each building on the last:\n\n\n1. Discovery\n\n\nFirst, it finds all your repositories:\n\n\n```bash\n\nREPOS=$(aws ecr describe-repositories --query\n'repositories[*].repositoryName' --output text)\n\n```\n\n\n2. Tag enumeration\n\n\nThen, for each repository, it gets all the tags:\n\n\n```bash\n\nTAGS=$(aws ecr describe-images --repository-name $repo --query\n'imageDetails[*].imageTags[]' --output text)\n\n```\n\n\n3. Transfer\n\n\nFinally, it handles the actual migration:\n\n\n```bash\n\ndocker pull\n${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${repo}:${tag}\n\ndocker tag\n${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${repo}:${tag}\n${CI_REGISTRY_IMAGE}/${repo}:${tag}\n\ndocker push ${CI_REGISTRY_IMAGE}/${repo}:${tag}\n\n```\n\n\n## What you get\n\n\nRemember that platform engineer who didn't want to spend weeks on migration?\nHere's what this solution delivers:\n\n\n- automated discovery and migration of all repositories and tags\n\n- consistent image naming between ECR and GitLab\n\n- error handling for failed transfers\n\n- clear logging for tracking progress\n\n\nInstead of writing scripts and babysitting the migration, the platform\nengineer could focus on more valuable work.\n\n\n## Usage\n\n\nGetting started is straightforward:\n\n\n1. Copy the `.gitlab-ci.yml` to your repository.\n\n2. Configure the AWS and GitLab variables.\n\n3. Set `BULK_MIGRATE` to \"true\" to start the migration.\n\n\n## Best practices\n\n\nThrough helping teams with their migrations, we've learned a few things:\n\n\n- Run during off-peak hours to minimize the impact on your team.\n\n- Keep an eye on the pipeline logs - they'll tell you if anything needs\nattention.\n\n- Don't decommission ECR until you've verified all images transferred\nsuccessfully.\n\n- For very large migrations, consider adding rate limiting to avoid\noverwhelming your network\n\n\nWe've open-sourced this pipeline in our public GitLab repository because we\nbelieve platform engineers should spend time building valuable\ninfrastructure, not copying container images. Feel free to adapt it for your\nneeds or ask questions about implementation.\n\n\n> #### Get started with this and other package components with our [CI/CD\nCatalog\ndocumentation](https://gitlab.com/explore/catalog/components/package).\n",[108,810,9,495,694,1016],"solutions architecture",{"slug":1018,"featured":90,"template":700},"automating-container-image-migration-from-amazon-ecr-to-gitlab","content:en-us:blog:automating-container-image-migration-from-amazon-ecr-to-gitlab.yml","Automating Container Image Migration From Amazon Ecr To Gitlab","en-us/blog/automating-container-image-migration-from-amazon-ecr-to-gitlab.yml","en-us/blog/automating-container-image-migration-from-amazon-ecr-to-gitlab",{"_path":1024,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1025,"content":1031,"config":1037,"_id":1039,"_type":13,"title":1040,"_source":15,"_file":1041,"_stem":1042,"_extension":18},"/en-us/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd",{"title":1026,"description":1027,"ogTitle":1026,"ogDescription":1027,"noIndex":6,"ogImage":1028,"ogUrl":1029,"ogSiteName":685,"ogType":686,"canonicalUrls":1029,"schema":1030},"Automating cybersecurity threat detections with GitLab CI/CD","Discover how GUARD automates cybersecurity threat detections through the use\nof GitLab CI/CD and how it ensures high-quality detections.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663239/Blog/Hero%20Images/AdobeStock_1023776629.jpg","https://about.gitlab.com/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Automating cybersecurity threat detections with GitLab CI/CD\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Mitra Jozenazemian\"}],\n        \"datePublished\": \"2025-01-29\",\n      }",{"title":1026,"description":1027,"authors":1032,"heroImage":1028,"date":1034,"body":1035,"category":830,"tags":1036},[1033],"Mitra Jozenazemian","2025-01-29","*This blog post is the second post in a series about [GitLab Universal\nAutomated Response and Detection\n(GUARD)](https://about.gitlab.com/blog/unveiling-the-guard-framework-to-automate-security-detections-at-gitlab/).*\n\n\nWriting and deploying security threat detections in an organization’s\nsecurity information event management platform (SIEM) is a critical\ncomponent of a successful cybersecurity program. Moving from manual\ndetection engineering to a fully automated process by implementing\nDetections as Code (DaC) ensures detection consistency, quality, auditing,\nand automated testing. At GitLab, we’ve embedded DaC capabilities into\nGUARD, our fully automated detection and response framework. \n\n\n## The problem: Source control and automated tests\n\n\nThe [Signals\nEngineering](https://handbook.gitlab.com/handbook/security/security-operations/signals-engineering/)\nand\n[SIRT](https://handbook.gitlab.com/handbook/security/security-operations/sirt/)\nteam at GitLab share the responsibility to create, update, and decommission\nthreat detections in our SIEM. Maintaining a single source of truth for\ndetections is critical to ensure detection consistency and quality standards\nare met. Our teams made the conscious decision to abstract the detection\ncreation process from our SIEM, improving our issue tracking, consistency,\nroll-back process, and metrics. Additionally, conducting pre-commit\ndetection tests outside of our SIEM ensured that newly created detections\ndidn’t introduce overly false positive heavy alerts, which would require\ntuning or disablement while the alert was fixed. \n\n\n## The Solution: Leverage GitLab CI/CD for detection testing and validation\n\n\nTo address these challenges, we developed an efficient workflow using GitLab\n[CI/CD](https://about.gitlab.com/topics/ci-cd/), resulting in a streamlined\nand secure SIEM detection deployment process.\n\n\n### Key components of the GUARD DaC pipeline \n\n\n__1. Detections stored in JSON format in a GitLab project__\n\n\nGitLab uses the JSON format for our threat detections. The template includes\nessential information such as SIEM query logic, detection title, and\ndescription along with runbook page link, MITRE tactic and technique related\nto the detection, and other necessary details.\n\n\n__2. Initiating merge requests__\n\n\nWhen a GitLab team member intends to create a new threat detection, update\nan existing one, or delete a current detection, they initiate the process by\nsubmitting a merge request (MR) in the DaC project containing the detection\nJSON template. Creating the MR automatically triggers a CI/CD pipeline.\n\n\n__3. Automated validation with CI/CD jobs__\n\n\nEach MR contains a number of automated checks via GitLab CI/CD:   \n\n* Query format validation queries SIEM API to ensure detection query is\nvalid  \n\n* JSON Detection fields validation validates all required fields are\npresent, and are in the correct format   \n\n* New detections and detection modification trigger a number of SIEM API\ncalls to ensure the detection does not have any errors and that no issues\nwill be introduced into our production detection rules   \n\n* Detection deletion MRs trigger the pipeline to issue a SIEM API query to\nensure the detection to be deleted is still active and can be deleted \n\n\n__4. Peer review and approval__\n\n\nWhen a detection MR job completes successfully, a peer review is required to\nreview and confirm the MR meets required quality and content standards\nbefore the detection MR can be merged. [Merge request approval\nrules](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/rules.html)\nare used to trigger the peer review process. \n\n\n__5. Merge and final deployment__\n\n\nAfter the MR is approved, it is merged into the main branch. As part of the\nCI/CD pipeline, an automated job executes a SIEM API command in order to\nperform two tasks:   \n\n* Create the new detection or update/delete the existing detection if\nneeded.   \n\n* Extract the MITRE ATT&CK tactic and technique information related to the\nalert from the JSON files and transmit these details to a lookup table\nwithin the SIEM. This lookup table plays an important role in mapping our\nalerts to MITRE tactics and techniques, helping us improve our threat\nanalysis and identify gaps in our detection capabilities in alignment with\nthe MITRE framework.\n\n\n**Note:** The necessary credentials for these actions are securely stored in\n[CI/CD variables](https://docs.gitlab.com/ee/ci/variables/) to ensure the\nprocess remains confidential and secure.\n\n\nBelow is a template GitLab CI/CD `gitlab-ci.yml` configuration file for a\nDaC pipeline: \n\n\n```\n\n\n#\n---------------------------------------------------------------------------\n#\n\n# GitLab CI/CD Pipeline for SIEM Detection Management\n\n#\n---------------------------------------------------------------------------\n#\n\n\nimage: python:3.12\n\n\n#\n---------------------------------------------------------------------------\n#\n\n# Global Configuration\n\n#\n---------------------------------------------------------------------------\n#\n\n\nbefore_script:\n  - apt-get update && apt-get install -y jq\n  - pip install --upgrade pip\n  - pip install -r requirements.txt\n\n#\n---------------------------------------------------------------------------\n#\n\n\nstages:\n  - fetch\n  - test\n  - process\n  - upload\n\n#\n---------------------------------------------------------------------------\n#\n\n# Fetch Stage\n\n#\n---------------------------------------------------------------------------\n#\n\n\nfetch_changed_files:\n  stage: fetch\n  Script:\n    - echo \"Fetching changed files...\"\n    - git branch\n    - git fetch origin $CI_DEFAULT_BRANCH:$CI_DEFAULT_BRANCH --depth 2000\n    - |\n      if [[ \"$CI_COMMIT_BRANCH\" == \"$CI_DEFAULT_BRANCH\" ]]; then\n        git diff --name-status HEAD^1...HEAD > changed-files-temp.txt\n      else\n        git fetch origin $CI_COMMIT_BRANCH:$CI_COMMIT_BRANCH --depth 2000\n        git diff --name-status ${CI_DEFAULT_BRANCH}...${CI_COMMIT_SHA} > changed-files-temp.txt\n      fi\n    - grep -E '\\.json$' changed-files-temp.txt > changed-files.txt || true\n    - flake8 .\n    - pytest\n  artifacts:\n    paths:\n      - changed-files.txt\n    expose_as: 'changed_files'\n\n#\n---------------------------------------------------------------------------\n#\n\n# Test Stage\n\n#\n---------------------------------------------------------------------------\n#\n\n\nflake8:\n  stage: test\n  script:\n    - echo \"Running Flake8 for linting...\"\n    - flake8 .\n\npytest:\n  stage: test\n  script:\n    - echo \"Running Pytest for unit tests...\"\n    - pytest\n  artifacts:\n    when: always\n    reports:\n      junit: report.xml\n\n#\n---------------------------------------------------------------------------\n#\n\n# Process Stage\n\n#\n---------------------------------------------------------------------------\n#\n\n\nprocess_files:\n  stage: process\n  script:\n    - echo \"Processing changed files...\"\n    - git clone --depth 2000 --branch $CI_DEFAULT_BRANCH $CI_REPOSITORY_URL\n    - mkdir -p modified_rules delete_file new_file\n    - python3 move-files.py -x changed-files.txt\n    - python3 check-alerts-format.py\n  artifacts:\n    paths:\n      - modified_rules\n      - delete_file\n      - new_file\n#\n---------------------------------------------------------------------------\n#\n\n# Upload Stage\n\n#\n---------------------------------------------------------------------------\n#\n\n\nupdate_rules:\n  stage: upload\n  script:\n    - echo \"Uploading updated rules and lookup tables...\"\n    - git fetch origin $CI_DEFAULT_BRANCH:$CI_DEFAULT_BRANCH --depth 2000\n    - git clone --depth 2000 --branch $CI_DEFAULT_BRANCH $CI_REPOSITORY_URL \n    - python3 update-rules.py\n    - python3 update-exceptions.py\n    - python3 create_ttps_layers.py\n  rules:\n    - if: $CI_COMMIT_BRANCH == \"main\" && $CI_PIPELINE_SOURCE != \"schedule\"\n      changes:\n        - detections/**/*\n        - exceptions/**/*\n```\n\n\nThe diagram below illustrates the workflow of the CI/CD process described\nabove.\n\n\n```mermaid\n\ngraph TD;\n    fetch[Fetch Stage: Identify Changed Files] --> test[Test Stage: Run Linting and Tests];\n    test --> process[Process Stage: Categorize Files];\n    process --> upload[Upload Stage: Update Rules and Lookup Tables];\n    fetch --> fetch_details[Details: Filter JSON files, Output 'changed-files.txt'];\n    test --> test_details[Details: Run Flake8 for linting, Pytest for testing];\n    process --> process_details[Details: Categorize into 'modified', 'new', 'deleted', Prepare for upload];\n    upload --> upload_details[Details: Update repo, Update detections in SIEM and SIEM lookup table];\n```\n\n\n## Benefits and outcomes\n\n\nAutomating our detections lifecycle through a DaC CI/CD-powered workflow\nintroduces numerous benefits to our threat detection deployment process:\n\n\n* Automation: Automating the creation and validation of SIEM detections\nreduces manual errors and saves time.\n\n* Enhanced security: The CI-driven workflow enforces a \"least privilege\"\npolicy, ensuring consistency, peer reviews, and quality standards for\ncreating, updating, or deleting threat detections. \n\n* Efficiency: The standardized JSON detection format and automated creation\nexpedite the deployment process.\n\n* Collaboration: The MR and review process fosters collaboration and\nknowledge sharing among GitLab team members.\n\n* Version control: Treating threat detection as code abstracts the\ndetections from the SIEM platform they are ultimately stored in. This\nabstraction provides a historical record of changes, facilitates\ncollaboration, and enables rollbacks to previous configurations if issues\narise.\n\n\n## Get started with DaC\n\n\nUsing GitLab CI/CD and a \"least privilege\" policy has made our SIEM\ndetection and alert management easier and more secure. Automation has\nimproved efficiency and reduced risks, providing a helpful example for\nothers wanting to improve their security and compliance. You can try this\ntutorial by signing up for a [free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).",[830,9,851,495,108],{"slug":1038,"featured":6,"template":700},"automating-cybersecurity-threat-detections-with-gitlab-ci-cd","content:en-us:blog:automating-cybersecurity-threat-detections-with-gitlab-ci-cd.yml","Automating Cybersecurity Threat Detections With Gitlab Ci Cd","en-us/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd.yml","en-us/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd",{"_path":1044,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1045,"content":1051,"config":1058,"_id":1060,"_type":13,"title":1061,"_source":15,"_file":1062,"_stem":1063,"_extension":18},"/en-us/blog/automating-with-gitlab-duo-part-1-generating-tests",{"title":1046,"description":1047,"ogTitle":1046,"ogDescription":1047,"noIndex":6,"ogImage":1048,"ogUrl":1049,"ogSiteName":685,"ogType":686,"canonicalUrls":1049,"schema":1050},"Automating with GitLab Duo, Part 1: Generating tests","Learn how we used the AI-driven DevSecOps platform to generate automated tests and improve our development speed and quality.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097480/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%284%29_3LZkiDjHLjhqEkvOvBsVKp_1750097480784.png","https://about.gitlab.com/blog/automating-with-gitlab-duo-part-1-generating-tests","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Automating with GitLab Duo, Part 1: Generating tests\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Byron Boots\"}],\n        \"datePublished\": \"2024-12-02\",\n      }",{"title":1046,"description":1047,"authors":1052,"heroImage":1048,"date":1054,"body":1055,"category":718,"tags":1056},[1053],"Byron Boots","2024-12-02","Automated testing is time-consuming and can feel like it’s not moving a project forward. However, as many developers have likely experienced, automated testing provides an overall positive return on investment. In building a custom module (we'll call it gitlab-helper for this article), this was particularly true.\n\nOur initial development focused on migrating tried and used functionality from existing scripts to a new module whose sole purpose was to serve as a baseline for future functionality. Although existing scripts lacked automated testing, their consistent usage was strong anecdotal evidence the functionality worked as expected.\n\nOur objective was to deliver a more mature solution to this problem, so automated testing became a necessity. This introduced the challenge of building efficiently, while balancing the time to test and ensure a robust product; and with a total of three team members, this was no small bottleneck. Therefore, the team decided to take advantage of [GitLab Duo](https://about.gitlab.com/gitlab-duo/), our suite of AI capabilities, for test generation, improving speed and quality of the delivered product.\n\nIn this three-part series on automating with GitLab Duo, we will cover:\n\n1. How we used GitLab Duo to generate tests for our code  \n2. How we worked interactively with GitLab Duo for more complex situations  \n3. The results we were able to achieve (Spoiler: 1 developer + GitLab Duo = 84% coverage in 2 days)\n\n## Using GitLab Duo to generate tests for code\n\nWhile functionality is available across tools, this article will cover using GitLab Duo in VS Code, with the [GitLab Workflow extension for VS Code](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow) to generate tests. Links to other GitLab Duo options are available in the [references](#references) below.\n\n### Install and enable GitLab Duo\n\nAs a prerequisite to using GitLab Duo, we ensured we had a GitLab Duo-enabled account. If you don't have GitLab Duo, you can [sign up for a free trial](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/?type=free-trial).\n\nTo use GitLab Duo Chat in VS Code, we followed the [instructions for installation](https://docs.gitlab.com/ee/user/gitlab_duo_chat/#use-gitlab-duo-chat-in-vs-code). Then, we were able to see the GitLab Duo Chat extension on the sidebar and open the Chat window.\n\n![Ask a question window](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097489/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097488918.png)\n\n### Generate tests with Chat\n\ngitlab-helper is a custom module built for standardizing interaction with the GitLab API across the team's work and extends other library functionalities to simplify development and scripting work. Once a method or feature was migrated to gitlab-helper and appeared to be implemented appropriately, the process to generate tests for it was simple:\n- Select the method, class, or entire file in the IDE.\n- Right-click on the selected code.\n- Under **GitLab Duo Chat**, select **Generate tests**.\n\n![Sequence to generate tests, including drop-down for generate tests](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097489/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097488919.png)\n\nWithin a few seconds, tests were generated and presented in the GitLab Duo Chat window. These tests can be reviewed and or added to the codebase, via copy/paste, into existing or new test files. As is the case with most natural language processing generations today, particularly around context, some of the initial tests created by GitLab Duo failed, thus requiring finetuning (for instance, when dealing with nested dependencies).\n\n> **Pro tip:** GitLab Duo does not auto-create files to add generated tests to. We found it was helpful to create new test files and add a `# Tests Generated by Duo` comment at the top of them and suffix them with `_duo.py` to indicate where the tests came from.\n\nGitLab Duo provided a great starting point for building out gitlab-helper’s automated testing and greatly improved test writing efficiency and code coverage, speeding up the development process substantially. Alongside GitLab Duo, numerous iterations of valuable tests were introduced into the gitlab-helper module with human oversight.\n\nRead the next installment in this series where we share [what we learned while using GitLab Duo for generating automated tests](https://about.gitlab.com/blog/automating-with-gitlab-duo-part-2-complex-testing/) and working interactively with AI for more complex situations.\n\n## References\n\nThere’s more than one way to use GitLab Duo to generate tests, check out the other options below:\n\n* The GitLab UI  \n* [The GitLab Web IDE (VS Code in the cloud)](https://docs.gitlab.com/ee/user/project/web_ide/index.html)  \n* VS Code, with the [GitLab Workflow extension for VS Code](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow)  \n* JetBrains IDEs, with the [GitLab Duo Plugin for JetBrains](https://plugins.jetbrains.com/plugin/22325-gitlab-duo)  \n* Visual Studio for Windows, with the [GitLab Extension for Visual Studio](https://marketplace.visualstudio.com/items?itemName=GitLab.GitLabExtensionForVisualStudio)\n",[720,9,1057,495,696],"testing",{"slug":1059,"featured":6,"template":700},"automating-with-gitlab-duo-part-1-generating-tests","content:en-us:blog:automating-with-gitlab-duo-part-1-generating-tests.yml","Automating With Gitlab Duo Part 1 Generating Tests","en-us/blog/automating-with-gitlab-duo-part-1-generating-tests.yml","en-us/blog/automating-with-gitlab-duo-part-1-generating-tests",{"_path":1065,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1066,"content":1072,"config":1079,"_id":1081,"_type":13,"title":1082,"_source":15,"_file":1083,"_stem":1084,"_extension":18},"/en-us/blog/aws-fargate-codebuild-build-containers-gitlab-runner",{"title":1067,"description":1068,"ogTitle":1067,"ogDescription":1068,"noIndex":6,"ogImage":1069,"ogUrl":1070,"ogSiteName":685,"ogType":686,"canonicalUrls":1070,"schema":1071},"Building containers with GitLab Runner & AWS Fargate executor","Build containers with the AWS Fargate Custom Executor for GitLab Runner and AWS CodeBuild","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667132/Blog/Hero%20Images/build-container-image-runner-fargate-codebuild-cover.jpg","https://about.gitlab.com/blog/aws-fargate-codebuild-build-containers-gitlab-runner","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to build containers with the AWS Fargate Custom Executor for GitLab Runner and AWS CodeBuild\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Elliot Rushton\"}],\n        \"datePublished\": \"2020-07-31\",\n      }",{"title":1073,"description":1068,"authors":1074,"heroImage":1069,"date":1076,"body":1077,"category":785,"tags":1078},"How to build containers with the AWS Fargate Custom Executor for GitLab Runner and AWS CodeBuild",[1075],"Elliot Rushton","2020-07-31","AWS Fargate does not allow containers to run in privileged mode. This means\nDocker-in-Docker (DinD), which enables the building and running of container\nimages inside of containers, does not work with the [AWS Fargate Custom\nExecutor driver for GitLab\nRunner](https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate).\nThe good news is that users don't have to be blocked by this and may use a\ncloud-native approach to build containers, effectively leveraging a seamless\nintegration with AWS CodeBuild in the [CI/CD pipeline](/topics/ci-cd/).\n\n\nWe provide in-depth instructions on how to autoscale GitLab CI on AWS\nFargate in [GitLab Runner's\ndocumentation](https://docs.gitlab.com/runner/configuration/runner_autoscale_aws_fargate/index.html).\nIn this blog post, we explain how to instrument CI containers and source\nrepositories to trigger AWS CodeBuild and use it to build container images.\n\n\n## Architecture overview\n\n\n![AWS Fargate + CodeBuild: a cloud-native approach to build containers with\nGitLab\nRunner](https://about.gitlab.com/images/blogimages/build-container-image-runner-fargate-codebuild.png)\n\nHow distinct CI workloads run on Fargate.\n\n{: .note.text-center}\n\n\nThe picture above illustrates distinct GitLab CI workloads running on\nFargate. The container identified by `ci-coordinator (001)` is running a\ntypical CI job which does not build containers, so it does not require\nadditional configuration or dependencies. The second container,\n`ci-coordinator (002)`, illustrates the problem to be tackled in this post:\nThe CI container includes the AWS CLI in order to send content to an Amazon\nS3 Bucket, trigger the AWS CodeBuild job, and fetch logs.\n\n\n## Prerequisites\n\n\nOnce these prerequisites are configured, you can dive into the six-step\nprocess to configure CI containers and source repositories to trigger AWS\nCodeBuild and use it to build container images.\n\n\n- The [AWS Fargate Custom Executor driver for GitLab\nRunner](https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate)\nmust be set-up appropriately.\n\n- Ensure the AWS IAM user permissions include the ability to create and\nconfigure S3 and CodeBuild resources.\n\n- AWS IAM user or service role with permissions to upload files to S3, start\nCodeBuild jobs, and read CloudWatch Logs.\n\n- AWS IAM user with permissions to create and configure IAM Policies and\nUsers.\n\n\n## Step 1: Create an AWS S3 bucket\n\n\n1. In the top menu of [AWS Management\nConsole](https://aws.amazon.com/console/) click Services.\n\n1. In the Storage section, select `S3`.\n\n1. Click `Create bucket`.\n\n1. Choose a descriptive name (`ci-container-build-bucket` will be used as\nexample) and select your preferred region.\n\n1. Leave all other fields with default values and click `Create bucket`.\n\n1. In the Buckets list, click the name of the bucket you created.\n\n1. Click `Create folder`.\n\n1. Give it the `gitlab-runner-builds` name.\n\n1. Click `Save`.\n\n\n## Step 2: Create an AWS CodeBuild Project\n\n\n1. Using the AWS Console, click `Services` in the top menu\n\n1. Select `CodeBuild` in the Developer Tools section\n\n1. Click `Create build project`\n\n1. In `Project Name` enter `ci-container-build-project`\n\n1. In `Source provider` select `Amazon S3`\n\n1. In `Bucket` select the `ci-container-build-bucket` created in step one\n\n1. In S3 object key or S3 folder enter `gitlab-runner-builds/build.zip`\n\n1. In `Environment image`, select `Managed image`\n\n1. For `Operating system` select your preferred OS from the available\noptions\n\n1. For `Runtime(s)`, choose `Standard`.\n\n1. For `Image`, select `aws/codebuild/standard:4.0`\n\n1. For `Image version`, select `Always use the latest image for this runtime\nversion`\n\n1. For `Environment type` select `Linux`\n\n1. Check the `Privileged` flag\n\n1. For the `Service role` select `New service role` and note the sugggested\n`Role name`\n\n1. For `Build specifications` select `Use a buildspec file`\n\n1. Scroll down to the bottom of the page and click \"Create build project\"\n\n\n## Step 3: Build the CI container image\n\n\nAs stated in Autoscaling GitLab CI on AWS Fargate, a [custom container is\nrequired](https://docs.gitlab.com/runner/configuration/runner_autoscale_aws_fargate/index.html#step-1-prepare-a-base-container-image-for-the-aws-fargate-task)\nto run GitLab CI jobs on Fargate. Since the solution relies on communicating\nwith S3 and CodeBuild, you'll need to [have the AWS CLI\ntool](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)\navailable in the CI container.\n\n\nInstall the `zip` tool to make S3 communication smoother. As an example of a\nUbuntu-based container, the lines below must be added to the CI container's\n`Dockerfile`:\n\n\n```dockerfile\n\nRUN apt-get update -qq -y \\\n    && apt-get install -qq -y curl unzip zip \\\n    && curl -Lo awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip \\\n    && unzip awscliv2.zip \\\n    && ./aws/install\n```\n\n\n## Step 4: Add CodeBuild configuration to the repository\n\n\nBy default, CodeBuild looks for a file named `buildspec.yml` in the build\nsource. This file will instruct CodeBuild on how to build and publish the\nresulting container image. Create this file with the content below and\ncommit it to the git repository (_if you changed the **Buildspec name** when\nconfiguring the CodeBuild project [in Step 2](#buildspec), please create the\nfile accordingly_):\n\n\n```yaml\n\nversion: 0.2\n\n\nphases:\n  install:\n    commands:\n      - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2&\n      - timeout 15 sh -c \"until docker info; do echo .; sleep 1; done\"\n  build:\n    commands:\n      - echo Build started on `date`\n      - docker -v\n      - docker build -t \u003CIMAGE-TAG> .\n      - echo Build completed on `date`\n```\n\n\n## Step 5: Set up the GitLab CI job\n\n\nNow we will set up the GitLab CI job that will pull everything together.\n\n\n### Interacting with CodeBuild through the AWS CLI\n\n\nThe CI job will need to interact with AWS Cloud to start CodeBuild jobs,\npoll the status of the jobs, and fetch logs. Commands such as `aws\ncodebuild` and `aws logs` help to tackle this, so let's use them in a\nscript, `codebuild.sh`:\n\n\n```bash\n\n#!/bin/bash\n\n\nbuild_project=ci-container-build-project\n\nbuild_id=$(aws codebuild start-build --project-name $build_project --query\n'build.id' --output text)\n\nbuild_status=$(aws codebuild batch-get-builds --ids $build_id --query\n'builds[].buildStatus' --output text)\n\n\nwhile [ $build_status == \"IN_PROGRESS\" ]\n\ndo\n    sleep 10\n    build_status=$(aws codebuild batch-get-builds --ids $build_id --query 'builds[].buildStatus' --output text)\ndone\n\n\nstream_name=$(aws codebuild batch-get-builds --ids $build_id --query\n'builds[].logs.streamName' --output text)\n\ngroup_name=$(aws codebuild batch-get-builds --ids $build_id --query\n'builds[].logs.groupName' --output text)\n\n\naws logs get-log-events --log-stream-name $stream_name --log-group-name\n$group_name --query 'events[].message' --output text\n\necho Codebuild completed with status $build_status\n\n```\n\n\n### Add a job to build the resulting container\n\n\nOnce the steps one through five are complete, the source repository will be\nstructured as follows:\n\n\n```plaintext\n\n/sample-repository\n  ├── .gitlab-ci.yml\n  ├── buildspec.yml\n  ├── codebuild.sh\n  ├── Dockerfile\n  ├── \u003CAPPLICATION-FILES>\n```\n\n\nThe final step to build the container is to add a job to `.gitlab-ci.yml`:\n\n\n```yaml\n\ndockerbuild:\n  stage: deploy\n  script:\n    - zip build.zip buildspec.yml Dockerfile \u003CAPPLICATION-FILES>\n    - aws configure set default.region \u003CREGION>\n    - aws s3 cp build.zip s3://ci-container-build-bucket/gitlab-runner-builds/build.zip\n    - bash codebuild.sh\n```\n\n\nBelow are some definitions from terms in the script:\n\n\n- `\u003CAPPLICATION-FILES>` is a placeholder for the files that will be required\nto successfully build the resulting container image using the `Dockerfile`,\ne.g., `package.json` and `app.js` in a Node.js application\n\n- `Dockerfile` is used to build the resulting image. _Note: It is not the\nsame file used to build the CI container image, mentioned in [Step 3: Build\nthe CI container image](#step-3-build-the-ci-container-image)_\n\n- Zip and AWS CLI must be installed in the CI container to make the script\nwork – refer to [Step 3: Build the CI container\nimage](#step-3-build-the-ci-container-image) for details\n\n\n## Step 6: Set up AWS credentials\n\n\nThe final step is to set up the AWS credentials. As we already mentioned,\nthe CI job will interact with AWS through the AWS CLI to perform a number of\noperations, and to do that, the AWS CLI needs to authenticate as an IAM user\nwith the permissions listed below. We recommend you create a new user and\ngrant it minimal privileges instead of using your personal AWS user account.\nFor the sake of simplicity, we suggest this approach to complete this\nwalk-through guide.\n\n\nThis AWS user only needs programmatic access and do not forget to make note\nof its Access key ID and Secret access key – they will be needed later. A\nsimple way to grant only the minimal privileges for the new user is to\ncreate a customer managed policy since it can be directly attached to the\nuser. A group might also be used to grant the same privileges for more\nusers, but it is not mandatory for running the sample workflow.\n\n\n- S3\n\n  ```json\n  {\n    \"Effect\": \"Allow\",\n    \"Action\": \"s3:PutObject\",\n    \"Resource\": \"arn:aws:s3:::ci-container-build-bucket/gitlab-runner-builds/*\"\n  }\n  ```\n\n- CodeBuild\n\n  ```json\n  {\n    \"Effect\": \"Allow\",\n    \"Action\": [\"codebuild:StartBuild\", \"codebuild:BatchGetBuilds\"],\n    \"Resource\": \"arn:aws:codebuild:\u003CREGION>:\u003CACCOUNT-ID>:project/ci-container-build-project\"\n  }\n  ```\n\n- CloudWatch Logs\n\n  ```json\n  {\n    \"Effect\": \"Allow\",\n    \"Action\": \"logs:GetLogEvents\",\n    \"Resource\": \"arn:aws:logs:\u003CREGION>:\u003CACCOUNT-ID>:log-group:/aws/codebuild/ci-container-build-project:log-stream:*\"\n  }\n  ```\n\nThe access credentials can be provided to AWS CLI through GitLab CI\nenvironment variables. Please go to your GitLab Project's **CI/CD\nSettings**, click **Expand** in the **Variables** section, add\n`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with the values you got from\nthe AWS Management Console after creating the IAM user. See the image below\nfor the result you can expect:\n\n\n![Providing AWS credentials for GitLab\nRunner](https://about.gitlab.com/images/blogimages/build-container-image-runner-fargate-codebuild-credentials.png)\n\n\nUsing an IAM Role and [Amazon ECS temporary/unique security\ncredentials](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html)\nis also possible, but not covered in this tutorial.\n\n{: .note.text-center}\n\n\n## Step 7: It's showtime\n\n\nWith all configurations in place, commit the changes and trigger a new\npipeline to watch the magic happen!\n\n\n### Just need the highlights?\n\n\n1. The CI job script added in [Step\n5](#add-a-job-to-build-the-resulting-container) compresses the resulting\ncontainer image build files into `build.zip`\n\n1. `build.zip` is then uploaded to the S3 Bucket we created in [Step 1:\nCreate an Amazon S3 Bucket](#step-1-create-an-amazon-s3-bucket)\n\n1. Next, `codebuild.sh` starts a CodeBuild job based on the project created\nin [Step 2: Create an AWS CodeBuild\nProject](#step-2-create-an-aws-codebuild-project) (Note: that project has an\nS3 object as its source provider)\n\n1. Finally, the CodeBuild job downloads `gitlab-runner-builds/build.zip`\nfrom S3, decompresses it and – from `buildspec.yml`– builds the resulting\ncontainer image\n\n\nA sample repository, demonstrating everything described in the article is\navailable\n[here](https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/codebuild-on-fargate-example/).\n\n\n## Cleanup\n\n\nIf you want to perform a cleanup after testing the custom executor with AWS\nFargate and CodeBuild, you should remove the following objects:\n\n\n- AWS S3 bucket created in [Step 1](#step-1-create-an-amazon-s3-bucket)\n\n- AWS CodeBuild project created in [Step\n2](#step-2-create-an-aws-codebuild-project)\n\n- `RUN` command added to the CI container image in [Step\n3](#step-3-build-the-ci-container-image)\n\n- The `buildspec.yml` file created in [Step\n4](#step-4-add-codebuild-configuration-to-the-repository)\n\n- The `codebuild.sh` file created in [Step\n5](#step-5-set-up-the-gitlab-ci-job)\n\n- The `dockerbuild` job added to `.gitlab-ci.yml` in [Step\n5](#step-5-set-up-the-gitlab-ci-job)\n\n- IAM policy, user (and maybe group) created in [Step\n6](#step-6-set-up-aws-credentials)\n\n- GitLab CI/CD variables in [Step 6](#step-6-set-up-aws-credentials)\n\n\nRead more about GitLab and AWS:\n\n-[How autoscaling GitLab CI works on AWS\nFargate](/blog/introducing-autoscaling-gitlab-runners-on-aws-fargate/)\n\n-[GitLab 12.10 released with Requirements Management and Autoscaling CI on\nAWS Fargate](/releases/2020/04/22/gitlab-12-10-released/)\n\n-[Announcing 32/64-bit Arm Runner Support for AWS\nGraviton2](/blog/gitlab-arm-aws-graviton2-solution/)\n\n\nCover image by [Lucas van Oort](https://unsplash.com/@switch_dtp_fotografie)\non [Unsplash](https://unsplash.com)\n\n{: .note}\n",[108,232,9],{"slug":1080,"featured":6,"template":700},"aws-fargate-codebuild-build-containers-gitlab-runner","content:en-us:blog:aws-fargate-codebuild-build-containers-gitlab-runner.yml","Aws Fargate Codebuild Build Containers Gitlab Runner","en-us/blog/aws-fargate-codebuild-build-containers-gitlab-runner.yml","en-us/blog/aws-fargate-codebuild-build-containers-gitlab-runner",{"_path":1086,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1087,"content":1093,"config":1101,"_id":1103,"_type":13,"title":1104,"_source":15,"_file":1105,"_stem":1106,"_extension":18},"/en-us/blog/basics-of-gitlab-ci-updated",{"title":1088,"description":1089,"ogTitle":1088,"ogDescription":1089,"noIndex":6,"ogImage":1090,"ogUrl":1091,"ogSiteName":685,"ogType":686,"canonicalUrls":1091,"schema":1092},"Running CI jobs in sequential, parallel, and custom orders","New to continuous integration? Learn how to build your first CI pipeline with GitLab.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662061/Blog/Hero%20Images/cicdcover.png","https://about.gitlab.com/blog/basics-of-gitlab-ci-updated","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The basics of CI: How to run jobs sequentially, in parallel, or out of order\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2020-12-10\",\n      }",{"title":1094,"description":1089,"authors":1095,"heroImage":1090,"date":1097,"body":1098,"category":785,"tags":1099,"updatedDate":1100},"The basics of CI: How to run jobs sequentially, in parallel, or out of order",[1096],"Itzik Gan Baruch","2020-12-10","Let's assume that you don't know anything about [continuous integration\n(CI)](/topics/ci-cd/) and [why it's\nneeded](/blog/how-to-keep-up-with-ci-cd-best-practices/) in the software\ndevelopment lifecycle.\n\n\nImagine that you work on a project, where all the code consists of two text\nfiles. Moreover, it is super critical that the concatenation of these two\nfiles contains the phrase \"Hello, world.\"\n\n\nIf it's not there, the whole development team won't get paid that month.\nYeah, it is that serious!\n\n\nThe most responsible software developer wrote a small script to run every\ntime we are about to send our code to customers.\n\n\nThe code is pretty sophisticated:\n\n\n```bash\n\ncat file1.txt file2.txt | grep -q \"Hello world\"\n\n```\n\n\nThe problem is that there are 10 developers on the team, and, you know,\nhuman factors can hit hard.\n\n\nA week ago, a new guy forgot to run the script and three clients got broken\nbuilds. So you decided to solve the problem once and for all. Luckily, your\ncode is already on GitLab, and you remember that there is [built-in\nCI](/solutions/continuous-integration/). Moreover, you heard at a conference\nthat people use CI to run tests...\n\n\n## Let's run our first test inside CI\n\n\nAfter taking a couple of minutes to find and read the docs, it seems like\nall we need is these two lines of code in a file called `.gitlab-ci.yml`:\n\n\n```yaml\n\ntest:\n  script: cat file1.txt file2.txt | grep -q 'Hello world'\n```\n\n\nWe commit it, and hooray! Our build is successful:\n\n\n![build\nsucceeded](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/build_succeeded.png)\n\n\nLet's change \"world\" to \"Africa\" in the second file and check what happens:\n\n\n![build\nfailed](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/build_failed.png)\n\n\nThe build fails as expected!\n\n\nOK, we now have automated tests here! GitLab CI will run our test script\nevery time we push new code to the source code repository in the DevOps\nenvironment.\n\n\n**Note:** In the above example, we assume that file1.txt and file2.txt exist\nin the runner host.\n\n\nTo run this example in GitLab, use the below code that first will create the\nfiles and then run the script.\n\n\n```yaml\n\ntest:\n\nbefore_script:\n      - echo \"Hello \" > | tr -d \"\\n\" | > file1.txt\n      - echo \"world\" > file2.txt\nscript: cat file1.txt file2.txt | grep -q 'Hello world'\n\n```\n\n\nFor the sake of compactness, we will assume that these files exist in the\nhost, and will not create them in the following examples.\n\n\n## Make results of builds downloadable\n\n\nThe next business requirement is to package the code before sending it to\nour customers. Let's automate that part of the software development process\nas well!\n\n\nAll we need to do is define another job for CI. Let's name the job\n\"package\":\n\n\n```yaml\n\ntest:\n  script: cat file1.txt file2.txt | grep -q 'Hello world'\n\npackage:\n  script: cat file1.txt file2.txt | gzip > package.gz\n```\n\n\nWe have two tabs now:\n\n\n![Two tabs - generated from two\njobs](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/two_tabs.png)\n\n\nHowever, we forgot to specify that the new file is a build _artifact_, so\nthat it could be downloaded. We can fix it by adding an `artifacts` section:\n\n\n```yaml\n\ntest:\n  script: cat file1.txt file2.txt | grep -q 'Hello world'\n\npackage:\n  script: cat file1.txt file2.txt | gzip > packaged.gz\n  artifacts:\n    paths:\n    - packaged.gz\n```\n\n\nChecking... it is there:\n\n\n![Checking the download\nbutton](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/artifacts.png)\n\n\nPerfect, it is! However, we have a problem to fix: The jobs are running in\nparallel, but we do not want to package our application if our tests fail.\n\n\n## Run jobs sequentially\n\n\nWe only want to run the 'package' job if the tests are successful. Let's\ndefine the order by specifying `stages`:\n\n\n```yaml\n\nstages:\n  - test\n  - package\n\ntest:\n  stage: test\n  script: cat file1.txt file2.txt | grep -q 'Hello world'\n\npackage:\n  stage: package\n  script: cat file1.txt file2.txt | gzip > packaged.gz\n  artifacts:\n    paths:\n    - packaged.gz\n```\n\n\nThat should be good!\n\n\nAlso, we forgot to mention, that compilation (which is represented by\nconcatenation in our case) takes a while, so we don't want to run it twice.\nLet's define a separate step for it:\n\n\n```yaml\n\nstages:\n  - compile\n  - test\n  - package\n\ncompile:\n  stage: compile\n  script: cat file1.txt file2.txt > compiled.txt\n  artifacts:\n    paths:\n    - compiled.txt\n\ntest:\n  stage: test\n  script: cat compiled.txt | grep -q 'Hello world'\n\npackage:\n  stage: package\n  script: cat compiled.txt | gzip > packaged.gz\n  artifacts:\n    paths:\n    - packaged.gz\n```\n\n\nLet's take a look at our artifacts:\n\n\n![Unnecessary\nartifact](https://about.gitlab.com/images/blogimages/the-basics-of-gitlab-ci/clean-artifacts.png)\n\n\nHmm, we do not need that \"compile\" file to be downloadable. Let's make our\ntemporary artifacts expire by setting `expire_in` to '20 minutes':\n\n\n```yaml\n\ncompile:\n  stage: compile\n  script: cat file1.txt file2.txt > compiled.txt\n  artifacts:\n    paths:\n    - compiled.txt\n    expire_in: 20 minutes\n```\n\n\nNow our config looks pretty impressive:\n\n\n- We have three sequential stages to compile, test, and package our\napplication.\n\n- We pass the compiled app to the next stages so that there's no need to run\ncompilation twice (so it will run faster).\n\n- We store a packaged version of our app in build artifacts for further\nusage.\n\n\n## Learning which Docker image to use\n\n\nSo far, so good. However, it appears our builds are still slow. Let's take a\nlook at the logs.\n\n\n![ruby3.1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/ruby-31.png)\n\n\nWait, what is this? Ruby 3.1?\n\n\nWhy do we need Ruby at all? Oh, GitLab.com uses Docker images to [run our\nbuilds](/blog/shared-runners/), and [by\ndefault](https://docs.gitlab.com/ee/user/gitlab_com/#shared-runners) it uses\nthe [`ruby:3.1`](https://hub.docker.com/_/ruby/) image. For sure, this image\ncontains many packages we don't need. After a minute of Googling, we figure\nout that there's an image called\n[`alpine`](https://hub.docker.com/_/alpine/), which is an almost blank Linux\nimage.\n\n\nOK, let's explicitly specify that we want to use this image by adding\n`image: alpine` to `.gitlab-ci.yml`.\n\n\nNow we're talking! We shaved nearly three minutes off:\n\n\n![Build speed\nimproved](https://about.gitlab.com/images/blogimages/the-basics-of-gitlab-ci/speed.png)\n\n\nIt looks like there are a lot of public images around:\n\n- [mysql](https://hub.docker.com/_/mysql/)\n\n- [Python](https://hub.docker.com/_/python/)\n\n- [Java](https://hub.docker.com/_/java/)\n\n- [php](https://hub.docker.com/_/php/)\n\n\nSo we can just grab one for our technology stack. It makes sense to specify\nan image that contains no extra software because it minimizes download time.\n\n\n## Dealing with complex scenarios\n\n\nSo far, so good. However, let's suppose we have a new client who wants us to\npackage our app into `.iso` image instead of `.gz`. Since CI does all the\nwork, we can just add one more job to it. ISO images can be created using\nthe [mkisofs](http://www.w3big.com/linux/linux-comm-mkisofs.html) command.\nHere's how our config should look:\n\n\n```yaml\n\nimage: alpine\n\n\nstages:\n  - compile\n  - test\n  - package\n\n# ... \"compile\" and \"test\" jobs are skipped here for the sake of compactness\n\n\npack-gz:\n  stage: package\n  script: cat compiled.txt | gzip > packaged.gz\n  artifacts:\n    paths:\n    - packaged.gz\n\npack-iso:\n  stage: package\n  script:\n  - mkisofs -o ./packaged.iso ./compiled.txt\n  artifacts:\n    paths:\n    - packaged.iso\n```\n\n\nNote that job names shouldn't necessarily be the same. In fact, if they were\nthe same, it wouldn't be possible to make the jobs run in parallel inside\nthe same stage of the software development process. Hence, think of same\nnames of jobs and stages as coincidence.\n\n\nAnyhow, the build is failing:\n\n\n![Failed build because of missing\nmkisofs](https://about.gitlab.com/images/blogimages/the-basics-of-gitlab-ci/mkisofs.png)\n\n\nThe problem is that `mkisofs` is not included in the `alpine` image, so we\nneed to install it first.\n\n\n## Dealing with missing software/packages\n\n\nAccording to the [Alpine Linux\nwebsite](https://pkgs.alpinelinux.org/contents?file=mkisofs&path=&name=&branch=edge&repo=&arch=)\n`mkisofs` is a part of the `xorriso` and `cdrkit` packages. These are the\nmagic commands that we need to run to install a package:\n\n\n```bash\n\necho \"ipv6\" >> /etc/modules  # enable networking\n\napk update                   # update packages list\n\napk add xorriso              # install package\n\n```\n\n\nFor CI, these are just like any other commands. The full list of commands we\nneed to pass to `script` section should look like this:\n\n\n```yml\n\nscript:\n\n- echo \"ipv6\" >> /etc/modules\n\n- apk update\n\n- apk add xorriso\n\n- mkisofs -o ./packaged.iso ./compiled.txt\n\n```\n\n\nHowever, to make it semantically correct, let's put commands related to\npackage installation in `before_script`. Note that if you use\n`before_script` at the top level of a configuration, then the commands will\nrun before all jobs. In our case, we just want it to run before one specific\njob.\n\n\n## Directed Acyclic Graphs: Get faster and more flexible pipelines\n\n\nWe defined stages so that the package jobs will run only if the tests\npassed. What if we want to break the stage sequencing a bit, and run a few\njobs earlier, even if they are defined in a later stage? In some cases, the\ntraditional stage sequencing might slow down the overall pipeline execution\ntime.\n\n\nImagine that our test stage includes a few more heavy tests that take a lot\nof time to execute, and that those tests are not necessarily related to the\npackage jobs. In this case, it would be more efficient if the package jobs\ndon't have to wait for those tests to complete before they can start. This\nis where Directed Acyclic Graphs (DAG) come in: To break the stage order for\nspecific jobs, you can define job dependencies which will skip the regular\nstage order.\n\n\nGitLab has a special keyword `needs`, which creates dependencies between\njobs, and allows jobs to run earlier, as soon as their dependent jobs\ncomplete.\n\n\nIn the below example, the pack jobs will start running as soon as the test\njob completes, so if, in future, someone adds more tests in the test stage,\nthe package jobs will start to run before the new test jobs complete:\n\n\n```yaml\n\npack-gz:\n  stage: package\n  script: cat compiled.txt | gzip > packaged.gz\n  needs: [\"test\"]\n  artifacts:\n    paths:\n    - packaged.gz\n\npack-iso:\n  stage: package\n  before_script:\n  - echo \"ipv6\" >> /etc/modules\n  - apk update\n  - apk add xorriso\n  script:\n  - mkisofs -o ./packaged.iso ./compiled.txt\n  needs: [\"test\"]\n  artifacts:\n    paths:\n    - packaged.iso\n```\n\n\nOur final version of `.gitlab-ci.yml`:\n\n\n```yaml\n\nimage: alpine\n\n\nstages:\n  - compile\n  - test\n  - package\n\ncompile:\n  stage: compile\n  before_script:\n      - echo \"Hello  \" | tr -d \"\\n\" > file1.txt\n      - echo \"world\" > file2.txt\n  script: cat file1.txt file2.txt > compiled.txt\n  artifacts:\n    paths:\n    - compiled.txt\n    expire_in: 20 minutes\n\ntest:\n  stage: test\n  script: cat compiled.txt | grep -q 'Hello world'\n\npack-gz:\n  stage: package\n  script: cat compiled.txt | gzip > packaged.gz\n  needs: [\"test\"]\n  artifacts:\n    paths:\n    - packaged.gz\n\npack-iso:\n  stage: package\n  before_script:\n  - echo \"ipv6\" >> /etc/modules\n  - apk update\n  - apk add xorriso\n  script:\n  - mkisofs -o ./packaged.iso ./compiled.txt\n  needs: [\"test\"]\n  artifacts:\n    paths:\n    - packaged.iso\n```\n\n\nWow, it looks like we have just created a pipeline! We have three sequential\nstages, the jobs `pack-gz` and `pack-iso`, inside the `package` stage, are\nrunning in parallel:\n\n\n![Pipelines\nillustration](https://about.gitlab.com/images/blogimages/the-basics-of-gitlab-ci/pipeline.png)\n\n\n## Elevating your pipeline\n\n\nHere is how to elevate your pipeline.\n\n\n### Incorporating automated testing into CI pipelines\n\n\nIn DevOps, a key software development strategy rule is making really great\napps with amazing user experience. So, let's add some tests in our CI\npipeline to catch bugs early in the entire process. This way, we fix issues\nbefore they get big and before we move on to work on a new project.\n\n\nGitLab makes our lives easier by offering out-of-the-box templates for\nvarious [tests](https://docs.gitlab.com/ee/ci/testing/). All we need to do\nis include these templates in our CI configuration.\n\n\nIn this example, we will include [accessibility\ntesting](https://docs.gitlab.com/ee/ci/testing/accessibility_testing.html):\n\n\n```yaml\n\nstages:\n  - accessibility\n\nvariables:\n  a11y_urls: \"https://about.gitlab.com https://www.example.com\"\n\ninclude:\n  - template: \"Verify/Accessibility.gitlab-ci.yml\"\n```\n\n\nCustomize the `a11y_urls` variable to list the URLs of the web pages to test\nwith [Pa11y](https://pa11y.org/) and [code\nquality](https://docs.gitlab.com/ee/ci/testing/code_quality.html).\n\n\n```yaml\n   include:\n   - template: Jobs/Code-Quality.gitlab-ci.yml\n```\n\n\nGitLab makes it easy to see the test report right in the merge request\nwidget area. Having the code review, pipeline status, and test results in\none spot makes everything smoother and more efficient.\n\n\n![Accessibility\nreport](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/Screenshot_2024-04-02_at_10.56.41.png)\n\n\u003Ccenter>\u003Ci>Accessibility merge request widget\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\n![Code quality widget in\nMR](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/Screenshot_2024-04-02_at_11.00.25.png)\n\n\u003Ccenter>\u003Ci>Code quality merge request widget\u003C/i>\u003C/center>\n\n\n### Matrix builds\n\n\nIn some cases, we will need to test our app in different configurations, OS\nversions, programming language versions, etc. For those cases, we'll use the\n[parallel:matrix](https://docs.gitlab.com/ee/ci/yaml/#parallelmatrix) build\nto test our application across various combinations in parallel using one\njob configuration. In this blog, we'll test our code with different Python\nversions using the matrix keyword.\n\n\n```yaml\n\npython-req:\n  image: python:$VERSION\n  stage: lint\n  script:\n    - pip install -r requirements_dev.txt\n    - chmod +x ./build_cpp.sh\n    - ./build_cpp.sh\n  parallel:\n    matrix:\n      - VERSION: ['3.8', '3.9', '3.10', '3.11']   # https://hub.docker.com/_/python\n```\n\n\nDuring pipeline execution, this job will run in parallel four times, each\ntime using different Python image as shown below:\n\n\n![Matrix job\nrunning](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/Screenshot_2024-04-02_at_11.12.48.png)\n\n\n### Unit testing\n\n\n#### What are unit tests?\n\n\nUnit tests are small, targeted tests that check individual components or\nfunctions of software to ensure they work as expected. They are essential\nfor catching bugs early in the software development process and verifying\nthat each part of the code performs correctly in isolation.\n\n\nExample: Imagine you're developing a calculator app. A unit test for the\naddition function would check if 2 + 2 equals 4. If this test passes, it\nconfirms that the addition function is working correctly.\n\n\n#### Unit testing best practices\n\n\nIf the tests fail, the pipeline fails and users get notified. The developer\nneeds to check the job logs, which usually contain thousands of lines, and\nsee where the tests failed so that they can fix them. This check is\ntime-consuming and inefficient.\n\n\nYou can configure your job to use [unit test\nreports](https://docs.gitlab.com/ee/ci/testing/unit_test_reports.html).\nGitLab displays reports on the merge request and on the pipelines details\npage, making it easier and faster to identify the failure without having to\ncheck the entire log.\n\n\n##### JUnit test report\n\n\nThis is a sample JUnit test report:\n\n\n![pipelines JUnit test report v13\n10](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674097/Blog/Content%20Images/pipelines_junit_test_report_v13_10.png){:\n.shadow.center}\n\n\n### Integration and end-to-end testing strategies\n\n\nIn addition to our regular development routine, it's super important to set\nup a special pipeline just for integration and end-to-end testing. This\nchecks that all the different parts of our code work together smoothly,\nincluding those\n[microservices](https://about.gitlab.com/topics/microservices/), UI testing,\nand any other components.\n\n\nWe run these tests\n[nightly](https://docs.gitlab.com/ee/ci/pipelines/schedules.html). We can\nset it up so that the [results automatically get sent to a special Slack\nchannel](https://docs.gitlab.com/ee/user/project/integrations/gitlab_slack_application.html#notification-events).\nThis way, when developers come in the next day, they can quickly spot any\nissues. It's all about catching and fixing problems early on!\n\n\n### Test environment\n\n\nFor some of the tests, we may need a test environment to properly test our\napps. With GitLab CI/CD, we can automate the deployment of testing\nenvironments and save a ton of time. Since this blog mostly focuses on CI, I\nwon't elaborate on this, but you can refer to this section in the [GitLab\ndocumentation](https://docs.gitlab.com/ee/topics/release_your_application.html).\n\n\n## Implementing security scans in CI pipelines\n\n\nHere are the ways to implement security scans in CI pipelines.\n\n\n### SAST and DAST integration\n\n\nWe're all about keeping our code safe. If there are any vulnerabilities in\nour latest changes, we want to know ASAP. That's why it's a good idea to add\nsecurity scans to your pipeline. They'll check the code with every commit\nand give you a heads up about any risks. We've put together a product tour\nto walk you through adding scans, including static application security\ntesting ([SAST](https://docs.gitlab.com/ee/user/application_security/sast/))\nand dynamic application security testing\n([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)), to\nyour CI pipeline.\n\n\n__Click__ the image below to start the tour.\n\n\n[![Scans product\ntour](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/Screenshot_2024-04-14_at_13.44.42.png)](https://gitlab.navattic.com/gitlab-scans)\n\n\nPlus, with AI, we can dig even deeper into vulnerabilities and get\nsuggestions on how to fix them. Check out this demo for more info.\n\n\n__Click__ the image below to start the tour.\n\n\n[![product tour explain vulnerability\n](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674096/Blog/Content%20Images/Screenshot_2024-04-14_at_13.50.24.png)](https://tech-marketing.gitlab.io/static-demos/pt-explain-vulnerability.html)\n\n\n## Recap\n\n\nThere's much more to cover but let's stop here for now. All examples were\nmade intentionally trivial so that you could learn the concepts of GitLab CI\nwithout being distracted by an unfamiliar technology stack. Let's wrap up\nwhat we have learned:\n\n\n1. To delegate some work to GitLab CI you should define one or more\n[jobs](https://docs.gitlab.com/ee/ci/jobs/) in `.gitlab-ci.yml`.\n\n2. Jobs should have names and it's your responsibility to come up with good\nones.\n\n3. Every job contains a set of rules and instructions for GitLab CI, defined\nby [special keywords](#keywords).\n\n4. Jobs can run sequentially, in parallel, or out of order using\n[DAG](https://docs.gitlab.com/ee/ci/directed_acyclic_graph/index.html).\n\n5. You can pass files between jobs and store them in build artifacts so that\nthey can be downloaded from the interface.\n\n6. Add [tests and security\nscans](https://docs.gitlab.com/ee/development/integrations/secure.html) to\nthe CI pipeline to ensure the quality and security of your app.\n\n\nBelow are more formal descriptions of the terms and keywords we used, as\nwell as links to the relevant documentation.\n\n\n### Keyword descriptions and documentation\n\n\n{: #keywords}\n\n\n| Keyword/term       | Description |\n\n|---------------|--------------------|\n\n| [.gitlab-ci.yml](https://docs.gitlab.com/ee/ci/yaml/) | File containing\nall definitions of how your project should be built |\n\n| [script](https://docs.gitlab.com/ee/ci/yaml/#script)        | Defines a\nshell script to be executed |\n\n| [before_script](https://docs.gitlab.com/ee/ci/yaml/#before_script) | Used\nto define the command that should be run before (all) jobs |\n\n|\n[image](https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-image)\n| Defines what Docker image to use |\n\n| [stages](https://docs.gitlab.com/ee/ci/yaml/#stages)         | Defines a\npipeline stage (default: `test`) |\n\n| [artifacts](https://docs.gitlab.com/ee/ci/yaml/#artifacts)     | Defines a\nlist of build artifacts |\n\n|\n[artifacts:expire_in](https://docs.gitlab.com/ee/ci/yaml/#artifactsexpire_in)\n| Used to delete uploaded artifacts after the specified time |\n\n| [needs](https://docs.gitlab.com/ee/ci/yaml/#needs) | Used to define\ndependencies between jobs and allows to run jobs out of order |\n\n| [pipelines](https://about.gitlab.com/topics/ci-cd/cicd-pipeline/) | A\npipeline is a group of builds that get executed in stages (batches) |\n\n\n## More on CI/CD\n\n\n- [GitLab’s guide to CI/CD for beginners](/blog/beginner-guide-ci-cd/)\n\n- [Get faster and more flexible pipelines with a Directed Acyclic\nGraph](/blog/directed-acyclic-graph/)\n\n- [Decrease build time with custom Docker\nimage](http://beenje.github.io/blog/posts/gitlab-ci-and-conda/)\n\n- [Introducing the GitLab CI/CD Catalog\nBeta](https://about.gitlab.com/blog/introducing-the-gitlab-ci-cd-catalog-beta/)\n\n\n## FAQ\n\n\n### How do you choose between running CI jobs sequentially vs. in parallel?\n\n\nConsiderations for choosing between running CI jobs sequentially or in\nparallel include job dependencies, resource availability, execution times,\npotential interference, test suite structure, and cost considerations. For\nexample, if you have a build job that must finish before a deployment job\ncan start, you would run these jobs sequentially to ensure the correct order\nof execution. On the other hand, tasks such as unit testing and integration\ntesting can typically run in parallel since they are independent and don't\nrely on each other's completion.\n\n\n### What are directed Acyclic Graphs in GitLab CI, and how do they improve\npipeline flexibility?\n\n\nA Directed Acyclic Graph (DAG) in GitLab CI breaks the linear order of\npipeline stages. It lets you set dependencies between jobs, so jobs in later\nstages start as soon as earlier stage jobs finish. This reduces overall\npipeline execution time, improves efficiency, and lets some jobs complete\nearlier than in a regular order.\n\n\n### What is the importance of choosing the right Docker image for CI jobs in\nGitLab?\n\n\nGitLab utilizes Docker images to execute jobs. The default image is\nruby:3.1. Depending on your job's requirements, it's crucial to choose the\nappropriate image. Note that jobs first download the specified Docker image,\nand if the image contains additional packages beyond what's necessary, it\nwill increase download and execution times. Therefore, it's important to\nensure that the chosen image contains only the packages essential for your\njob to avoid unnecessary delays in execution.\n\n\n## Next steps\n\n\nAs a next step and to further modernize your software development practice,\ncheck out the [GitLab CI/CD\nCatalog](https://docs.gitlab.com/ee/architecture/blueprints/ci_pipeline_components/)\nto learn how to standardize and reuse CI/CD components.\n",[787,9],"2024-04-24",{"slug":1102,"featured":6,"template":700},"basics-of-gitlab-ci-updated","content:en-us:blog:basics-of-gitlab-ci-updated.yml","Basics Of Gitlab Ci Updated","en-us/blog/basics-of-gitlab-ci-updated.yml","en-us/blog/basics-of-gitlab-ci-updated",{"_path":1108,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1109,"content":1115,"config":1122,"_id":1124,"_type":13,"title":1125,"_source":15,"_file":1126,"_stem":1127,"_extension":18},"/en-us/blog/beginner-guide-python-programming",{"title":1110,"description":1111,"ogTitle":1110,"ogDescription":1111,"noIndex":6,"ogImage":1112,"ogUrl":1113,"ogSiteName":685,"ogType":686,"canonicalUrls":1113,"schema":1114},"How to get started with Python programming","Python is increasingly popular, and for good reason. Here's our beginner's guide.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664962/Blog/Hero%20Images/python.jpg","https://about.gitlab.com/blog/beginner-guide-python-programming","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to get started with Python programming\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2021-10-21\",\n      }",{"title":1110,"description":1111,"authors":1116,"heroImage":1112,"date":1117,"body":1118,"category":1119,"tags":1120},[868],"2021-10-21","Are you a programming enthusiast who wants to learn Python? Are you new to coding? Do you need help deciding where to begin with Python? If you are looking for answers to these questions, then you are in the right place.\n\n## How to start writing code with Python\n\nPython is an easy-to-learn, easy-to-use and easy-to-deploy programming language, with rampant usage in building web and desktop applications, analyzing data and performing [DevOps](https://about.gitlab.com/topics/devops/) tasks. It is a free, open-source, object-oriented coding language used to write simple scripts and complex programs. Of the almost 700 programming languages, Python is considered one of the best to learn first.\n\n## Installing Python\n\nBefore discussing the basics of Python, it is essential to download and install Python on your desktop/laptop. Python works on multiple platforms, including Linux, Windows and Mac. It comes preinstalled on most Mac and Linux systems; however, you should download the latest version from the official Python website.\n\nTo check the current Python version on your system, open the command line and type “python -V”. \n\n![command prompt](https://about.gitlab.com/images/blogimages/python1.png){: .shadow}\n\nIf you have an outdated version, download either the 32- or 64-bit setup from the website based on your system requirements.\n\nThere are other alternatives for downloading the setup: for Windows, you can install it directly from Microsoft. For Linux, install it using the package manager. For macOS, you can download it from Homebrew.\n\nOnce the setup is downloaded, run the file installer, and click on “Install Now”. Once the installation is complete, you are ready to go. Below is an example of a Python installation for Windows.\n\n![install Python](https://about.gitlab.com/images/blogimages/python2.png){: .shadow}\n\n## Running Python in command prompt\n\nTo verify Python is installed and working correctly in Windows, open the command prompt and enter “python”, which will invoke the interpreter. You can directly execute Python codes in it.  For example, type “2*5+1” and press “enter”. You will see “11” as the output. Entering “quit ()” will exit the interpreter.\n\n![Python interpreter](https://about.gitlab.com/images/blogimages/python3.png){: .shadow}\n\n## Running Python in IDE\n\nWith the latest Python installed, you are now ready to start programming in Python. When writing long scripts or programs in Python, use Python’s built-in Integrated Development and Learning Environment (IDLE).\n\nStart the IDLE and then, from the File dropdown, select “New File”, which opens a new editing window. So now, on your screen, you have two windows: a Python shell and an untitled file.\n\n![Python shell and untitled file](https://about.gitlab.com/images/blogimages/python4.png){: .shadow}\n\nThe Python shell is a REPL environment, which is shorthand for \"read-eval-print loop\". It runs snippets of the code, usually one statement at a time. For example, by repeating the same calculation “2*5+1” that we did in the command prompt, you can see how a Python shell can function as a calculator.\n\n![Python as a calculator](https://about.gitlab.com/images/blogimages/python5.png){: .shadow}\n\nThe untitled window is a text editing window for writing complete programs. The shell displays its output. For example, the conventional first program of Python for beginners is printing “Hello World!”. Make sure you save the text editor before running it by pressing “F5”.\n\n![Hello World](https://about.gitlab.com/images/blogimages/python61.png){: .shadow}\n\n## The basics of Python\n\nWe know you can’t wait to start writing long scripts for games and websites, but you still have a long way to get there. Just like with learning any other language, you must first understand the basics of Python. \n\nThe **print()** function, as seen in the Hello World! example, prints a value on the output window. A value is the most basic thing a program uses. It can be a string, a numeric value or any other Python object. Any object within single/double quotations is called a string. For instance, the “Hello World!” that is printed in the above program is also of the type string. Numeric values like 4 and 4.5 are the types of integers and floats, respectively. You can change an integer or float into a string and vice versa using the built-in functions **int()**, **float()** and **str()**.\n\n![value in an output window](https://about.gitlab.com/images/blogimages/python7.png){: .shadow}\n\n## Python’s vocabulary\n\nPython is the simplest coding language. It is easy to read and understand. Unlike human languages, Python has a small vocabulary or reserved words holding special meaning. Terms other than this reserved vocabulary hold meaning only to you and are called variables. These 35 reserved words are:\n\n![Python terms](https://about.gitlab.com/images/blogimages/python8.png){: .shadow}\n\nMake sure you use these words for their specified purpose to avoid confusing the Python interpreter and causing a syntax error.\n\n### Naming variables\n\nSometimes you want to store values in your code for retrieving them later, which you can do by giving them symbolic names called variables. As seen below, we ask Python to store 5 and 6 with labels x and y, respectively, and then retrieve them later to find their sum.\n\n![storing variables](https://about.gitlab.com/images/blogimages/python9.png){: .shadow}\n\nThere are rules for choosing a name for a variable; failing to follow these gives a syntax error. A few mandatory rules are narrated below:\n\n1. The name can contain both letters and numbers, but it can’t start with a number.\n1. An underscore can appear in the name to separate multiple words.\n1. Special symbols like @#$ are illegal and should not appear in the name.\n1. Python keywords should not be used as names for variables.\n\n### Understanding operators and operands\n\nPython uses special symbols called “operators” for representing basic mathematical computation. The values to which these operators are applied are called operands. The symbols used as operators for subtraction, addition, division, multiplication and exponentiation are  -,+, /, * and **, respectively. \n\n![symbols for operators](https://about.gitlab.com/images/blogimages/python10.png){: .shadow}\n\nThe modulus operator (%) outputs the remainder of the first operand divided by the second operand. It is useful in checking whether a number is divisible by another and extracting the rightmost digit/digits of a number.\n\n![modulus operator](https://about.gitlab.com/images/blogimages/python11.png){: .shadow}\n\n### Using expressions\n\nA combination of values, variables and operators is called an expression. An expression typed in the shell gets evaluated, and the answer is displayed. However, in a script, an expression doesn't do anything on its own.\n\nPython uses the mathematical convention PEMDAS for the operators, which means that P for Parentheses has the highest precedence, then Exponentiation, Multiplication and Division, which have the same priority. Addition and Subtraction come next and also have the same precedence. Operators that have the same preference are also evaluated from left to right.\n\n![PEMDAS](https://about.gitlab.com/images/blogimages/python12.png){: .shadow}\n\nThe Addition and Multiplication operators also work with strings for concatenation and repeating a string, respectively.\n\n![addition and multiplication operators](https://about.gitlab.com/images/blogimages/python13.png){: .shadow}\n\nPython also allows you to take the value for a variable from the user via their keyboard. This can be done using a built-in function called **input**.\n\n![input](https://about.gitlab.com/images/blogimages/python14.png){: .shadow}\n\n## Write your first program\n\nNow it's time to write a short program using everything you've learned here. Write a script that takes two numbers as input and adds them. Do this on your own and see the code below to tally your work.\n\n![write a short program](https://about.gitlab.com/images/blogimages/python15.png){: .shadow}\n\n**Congratulations!** You just wrote your first program.\n\nLearning Python is easy and fun. We just helped you make it through the basics. To become a professional Python Programmer, you still have a lot to learn and practice. Good luck on your journey to becoming an expert coder.\n\nPhoto by \u003Ca href=\"https://unsplash.com/@davidclode?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">David Clode\u003C/a> on \u003Ca href=\"https://unsplash.com/s/photos/python?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">Unsplash\u003C/a>","devsecops",[789,1121,9],"careers",{"slug":1123,"featured":6,"template":700},"beginner-guide-python-programming","content:en-us:blog:beginner-guide-python-programming.yml","Beginner Guide Python Programming","en-us/blog/beginner-guide-python-programming.yml","en-us/blog/beginner-guide-python-programming",{"_path":1129,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1130,"content":1136,"config":1142,"_id":1144,"_type":13,"title":1145,"_source":15,"_file":1146,"_stem":1147,"_extension":18},"/en-us/blog/build-a-new-website-in-a-few-easy-steps-with-gitlab-pages",{"title":1131,"description":1132,"ogTitle":1131,"ogDescription":1132,"noIndex":6,"ogImage":1133,"ogUrl":1134,"ogSiteName":685,"ogType":686,"canonicalUrls":1134,"schema":1135},"Build a new website in a few easy steps with GitLab Pages ","This tutorial shows you how to create and host your personal website using GitLab Pages with a ready-to-use template that you can customize in minutes.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097716/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%281%29_7c3TDgNgct9xQbmTJSw0de_1750097716096.png","https://about.gitlab.com/blog/build-a-new-website-in-a-few-easy-steps-with-gitlab-pages","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Build a new website in a few easy steps with GitLab Pages \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Alex Fracazo\"}],\n        \"datePublished\": \"2025-03-03\",\n      }",{"title":1131,"description":1132,"authors":1137,"heroImage":1133,"date":1139,"body":1140,"category":694,"tags":1141},[1138],"Alex Fracazo","2025-03-03","A personal website is more than just a utility for digital creators and professionals in tech. It's a representation of your brand. But creating one from scratch can be time-consuming and expensive.\n\nWith [GitLab Pages](https://docs.gitlab.com/user/project/pages/), you can host your website with built-in features, including SSL certificates and a GitLab-provided domain. All of this is available on GitLab's free tier, making it an efficient solution for hosting your professional presence.\n\nWe're going to take you on a fun journey to craft a stunning personal website using GitLab Pages! We’ve got a super simple, versatile template that you can easily jazz up to reflect your unique style. So grab your favorite snack, get comfy, and let’s turn your online presence into something truly fabulous!\n\n## Prerequisites\n\nYou will need the following prerequisites before getting started:\n\n* A GitLab account (the [free tier](https://about.gitlab.com/pricing/) is sufficient)  \n* Basic familiarity with HTML/CSS  \n* Content and images you want to add to your website (optional)\n\nOnce you’re set up with a GitLab account and have your content handy, you can move on to the next steps.\n\n## Step 1: Create a new project\n\n1. Sign on to your GitLab account and create a project.\n\n![GitLab Pages tutorial - welcome screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097724/Blog/Content%20Images/Blog/Content%20Images/Capture-2025-02-27-183716_aHR0cHM6_1750097724662.png)\n\n2. Click **Create blank project**.\n\n![GitLab Pages tutorial - Create new project screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/Capture-2025-02-27-183814_aHR0cHM6_1750097724663.png)\n\n3. Fill in your project details:\n    * Name your project `yourusername.gitlab.io`. Replace `yourusername` with your GitLab username. **Tip:** The project name determines your website’s URL. If you name your project `yourusername.gitlab.io`, your website will be available at `https://yourusername.gitlab.io` with no additional path. However, if you use any other project name, your site will be available at `https://yourusername.gitlab.io/project-name`.\n    * Make the project public.\n4. Click **Create project**.\n\n![GitLab Pages tutorial - Create blank project screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097724666.png)\n\n![GitLab Pages tutorial - customized get started page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097724668.png)\n\n## Step 2: Add the template files\n\nStart by creating two new files in your repository:\n\n![GitLab Pages tutorial - Add new files to personal page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image13_aHR0cHM6_1750097724669.png)\n\n1. First, create `index.html`:\n    * In your project, click the **+** button and select **New file**.\n    * Name the file `index.html`.\n![GitLab Pages tutorial - new file page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image14_aHR0cHM6_1750097724671.png)\n    * Add your HTML content.\n        * Use the example HTML provided below. (Pro tip: Users can ask GitLab Duo Chat to generate HTML for enhanced functionality.)\n\n```    \n\u003C!DOCTYPE html>\n\u003Chtml>\n\u003Chead>\n    \u003Cmeta charset=\"utf-8\"/>\n    \u003Ctitle>[Your Name] - [Your Title]\u003C/title>\n    \u003Cmeta name=\"description\" content=\"[Your Name] is a [Your Title].\"/>\n    \u003Cmeta name=\"author\" content=\"[Your Name]\"/>\n    \u003Cmeta property=\"og:title\" content=\"[Your Name]\" />\n    \u003Cmeta property=\"og:description\" content=\"[Your Title]\" />\n    \u003Cmeta property=\"og:image\" content=\"og.png\" />\n    \u003Cmeta name=\"viewport\" content=\"width=device-width,initial-scale=1\"/>\n    \u003Clink href=\"https://unpkg.com/basscss@8.0.2/css/basscss.min.css\" rel=\"stylesheet\">\n    \u003Clink href=\"style.css\" rel=\"stylesheet\">\n    \u003Clink rel=\"shortcut icon\" type=\"image/png\" href=\"favicon.png\"/>\n\u003C/head>\n\u003Cbody>\n\u003Cdiv class=\"content\" id=\"content\">\n  \u003Cdiv class=\"p2 sm-p4 mt2 sm-mt4 mb2 sm-mb4\">  \n  \u003Cdiv class=\"fade mt3\">\n    \u003Ca target=\"_new\" href=\"[Your Linkedin URL]\">\n      \u003Cimg class=\"photo\" src=\"profile.png\" width=\"64\" height=\"64\">\n    \u003C/a>\n  \u003C/div>\n  \u003Ch2 class=\"mb0 mt4 fade\">\n    Hello, I'm [Your Name] \n    \u003Cspan class=\"smallcaps\">(\u003C/span>\n    \u003Ca target=\"_new\" href=\"[Your Linkedin URL]\">@[Your Handle]\u003C/a>\n    \u003Cspan class=\"smallcaps\">)\u003C/span>\n  \u003C/h2>\n  \u003Ch2 class=\"mt0 mb4 fade gray\">\n    I'm a [Your Title]\n  \u003C/h2>\n  \u003Cp class=\"mb4 fade\">\n    I'm a [Your Role] at [Your Company], [Brief company description].\n  \u003C/p>\n  \u003Cdiv class=\"fade\">\n    \u003Cp class=\"fade mb4\">\n      Your personal statement about what you do and what you're interested in. Add your contact preferences here.\n    \u003C/p>\n  \u003C/div>\n  \u003Cp class=\"fade mb4\">\n    \u003Cspan class=\"gray\">—\u003C/span> \n    [Your Name] \n    \u003Cspan class=\"smallcaps>(\u003C/span>\n    \u003Ca target=\"_new\" href=\"[Your Linkedin URL]\">@[Your Handle]\u003C/a>\n    \u003Cspan class=\"smallcaps\">)\u003C/span>\n  \u003C/p>\n  \u003C/div>\n\u003C/div>\n\u003C/body>\n\u003C/html> \n```\n\n* Add a commit message (e.g., \"Added index.html\").\n  * Click **Commit changes**.\n\n2. Create `style.css` (follow same steps above).\n\n```\nbody {\n  margin: 0;\n  padding: 0;\n  background: #000;\n  color: #f4f4f4;\n  font-family: \"Graphik Web\", system-ui, -apple-system, BlinkMacSystemFont, \"Helvetica Neue\", \"Helvetica\", \"Segoe UI\", Roboto, Ubuntu, sans-serif;\n  font-weight: 400;\n  font-smooth: antialiased;\n  -webkit-font-smoothing: antialiased;\n  -moz-osx-font-smoothing: grayscale;\n}\n\na {\n  color: #ff310a;\n  text-decoration: none;\n}\n\na:hover {\n  color: #CFEF54\n}\n\n.content {\n  max-width: 40rem;\n  margin: 0 auto;\n}\n\nimg.photo {\n  border-radius: 50%;\n}\n\np {\n  font-size: 1.5rem;\n  line-height: 1.4;\n  margin: 0;\n  letter-spacing: -0.05rem;\n}\n\nh2 {\n  font-weight: 400;\n  line-height: 1.3;\n  letter-spacing: -0.05rem;\n}\n\n.smallcaps {\n  font-variant: small-caps;\n  color:#333;\n}\n\n.gray{\n  color: #999;\n}\n\n.preloader {\n  display: flex;\n  justify-content: center;\n  align-items: center;\n  height: 100vh;\n  height: -moz-available;\n  height: -webkit-fill-available;\n  height: fill-available;\n  width: 100%;\n  background: #000;\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: 9999;\n  transition: opacity 0.3s linear;\n  transform: translate3d(0, 0, 0);\n}\n\nbody.loaded .preloader {\n  opacity: 0;\n}\n\n.fade {\n  animation: fadeIn 1s ease-in-out both;\n}\n\n.fade:nth-child(2) {\n\tanimation-delay: 1s;\n}\n\n.fade:nth-child(3) {\n\tanimation-delay: 2s;\n}\n\n.fade:nth-child(4) {\n\tanimation-delay: 3s;\n}\n\n.fade:nth-child(5) {\n\tanimation-delay: 4s;\n}\n\n.fade:nth-child(6) {\n\tanimation-delay: 5s;\n}\n\n.fade:nth-child(7) {\n\tanimation-delay: 6s;\n}\n\n.fade:nth-child(8) {\n\tanimation-delay: 7s;\n}\n\n.fade:nth-child(9) {\n\tanimation-delay: 8s;\n}\n\n.fade:nth-child(10) {\n\tanimation-delay: 9s;\n}\n\n.fade:nth-child(11) {\n\tanimation-delay: 10s;\n}\n\n.fade:nth-child(12) {\n\tanimation-delay: 11s;\n}\n\n.fade:nth-child(13) {\n\tanimation-delay: 12s;\n}\n\n@keyframes fadeIn {\n\tfrom {\n\t\topacity: 0;\n\t\ttransform: translate3d(0, 0%, 0);\n\t}\n\tto {\n\t\topacity: 1;\n\t\ttransform: translate3d(0, 0, 0);\n\t}\n} \n\n```\n\n## Step 3: Configure GitLab CI file\n\nThere are two ways to create the GitLab CI configuration file that tells GitLab how to build and deploy your site:\n\n![GitLab Pages tutorial - optimize your workflow with CI/CD pipelines screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097724672.png)\n\n**Option 1: Use Pipeline Editor (recommended)**\n\n1. Go to your project's **Build > Pipeline Editor**.\n\n![GitLab Pages tutorial - pipeline editor/main branch](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image12_aHR0cHM6_1750097724673.png)\n\n2. The `.gitlab-ci.yml` file will be automatically created. \n3. Copy and paste the following configuration: \n\n```\npages:\n  stage: deploy\n  script:\n    - mkdir .public\n    - cp -r * .public\n    - mv .public public\n  artifacts:\n    paths:\n      - public\n  only:\n    - main\n```\n\n![GitLab Pages Tutorial - New file in window](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097724674.png)\n\n**Option 2: Manual creation**\n\nIf you prefer to create the file manually: \n1. Create a new file named `.gitlab-ci.yml`. \n2. Add the following configuration:\n\n```\npages:\n  stage: deploy\n  script:\n    - mkdir .public\n    - cp -r * .public\n    - mv .public public\n  artifacts:\n    paths:\n      - public\n  only:\n    - main\n```\n\nThe key to getting your site running is the GitLab CI configuration file. This file tells GitLab how to build and deploy your site.\n\nLet's break down what each part does:\n\n**The script part**\n\n```\nscript:\n  - mkdir .public\n  - cp -r * .public\n  - mv .public public\n```\n\nThis creates a folder called `public` and copies all your website files into it. GitLab Pages uses this folder to serve your website by default, though you can [customize the publishing folder](https://docs.gitlab.com/user/project/pages/introduction/#customize-the-default-folder) if needed.\n\n**The only part**\n\n```\nonly:\n  - main\n\n```\n\nThis tells GitLab to only update your website when changes are made to the main branch. This helps prevent accidental updates from experimental changes.\n\n## Step 4: Watch the magic happen\n1. Commit all your changes.\n2. Go to **Build > Pipelines** to watch your deployment.\n3. Wait for the pipeline to complete successfully (indicated by a green checkmark).\n\n![GitLab Pages tutorial - pipeline running for new page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097724676.png)\n\n![GitLab Pages tutorial - pipeline passed for new page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097724677.png)\n\n## Step 5: Access your website\n\nOnce the pipeline completes successfully, your website will be available at: **https://[yourusername].gitlab.io/** .\n\nYou can find an overview of your deployed website and additional settings in your project's **Deploy > Pages** section. Here you'll find useful information. including: \n\n* Your website's access URLs   \n* Domain settings  \n  * By default GitLab enables **Unique domain**. Make sure to disable it if you want to use the GitLab-provided domain. Learn more with the [unique domain documentation](https://docs.gitlab.com/ee/user/project/pages#unique-domains).  \n* HTTPS certificates status   \n* Recent deployments   \n* Additional configuration options\n* Custom domains\n\nThis section is particularly helpful when setting up custom domains or troubleshooting deployment issues.\n\n**Customize your site**\n\n![GitLab Pages tutorial - customize site](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097724678.png)\n\n1. Replace all “Your ...” placeholders in `index.html` with your information.\n\n![GitLab Pages tutorial - upload file to customize page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097725/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750097724679.png)\n\n2. Add your images:\n    - profile.png - your profile photo (64x64px)\n    - favicon.png - your site favicon (32x32px)\n    - Og.png - OpenGraph image for social media preview (1200x630px)\n\n**See it in action**\n\nIf you're familiar with GitLab, feel free to [fork my repository](https://gitlab.com/fracazo/fracazo.gitlab.io) to get started quickly. \n\nHere is the final result:\n[https://fracazo.gitlab.io/](https://fracazo.gitlab.io/)\n\n**Common issues and solutions**\n- By default, GitLab enables \"Unique domain\" for Pages projects. To use the simpler GitLab-provided domain (like `username.gitlab.io`), go to **Deploy > Pages** and disable the \"Use unique domain\" option. While unique domains offer some technical advantages, like better asset path handling, you might prefer the cleaner URL structure for a personal website.\n- If your pipeline fails, check that you're using `main` instead of `master` in your `.gitlab-ci.yml` file.\n- Ensure your group and project is public for GitLab Pages to work.\n- If any jobs fail in your pipeline, you can check the job log for detailed error messages to help with troubleshooting.\n\nWith GitLab Pages and this template, you can have a professional/personal website up and running in minutes. The template is clean, responsive, and easy to customize. As you grow professionally, you can easily update your site directly through GitLab. \n\nYou can automate the deployment process by leveraging GitLab's CI/CD capabilities and focusing on creating great content.\n\nThe best part? All of this is available on GitLab's free tier, making it an excellent option for free hosting of your personal projects, documentation sites, or even small business websites. For more advanced features and configurations, check out our [Pages documentation](https://docs.gitlab.com/ee/user/project/pages/).\n\n## What’s next for GitLab Pages?\nWe're constantly working to make GitLab Pages even better for creators and developers. Here are some exciting improvements coming soon: \n\n### Simplified domain management \nWe have some exciting updates coming to GitLab Pages that will make managing your domains even easier and more fun! You can look forward to a streamlined dashboard that brings all your domain settings together in one friendly space, making everything easily accessible. \n\nYou’ll stay informed with real-time updates on your DNS and SSL certificate statuses, helping you keep your domains secure and running smoothly. \n\n### Custom domain setup\nSetting up custom domains will be a breeze with our easy-to-follow process, guiding you every step of the way. Plus, you'll be able to set up your custom domains to automatically redirect visitors from your old website address to your new one – perfect for when you want all your traffic to go to one main website. Learn more about [custom domains](https://docs.gitlab.com/ee/user/project/pages/custom_domains_ssl_tls_certification/index.html#set-up-a-custom-domain).\n\n> Get started with GitLab Pages today with [GitLab's free tier](https://about.gitlab.com/pricing/)! \n\n## Learn more\n- [GitLab Pages features review apps and multiple website deployment](https://about.gitlab.com/blog/gitlab-pages-features-review-apps-and-multiple-website-deployment/)\n- [GitLab Pages: Multiple website deployment documentation](https://docs.gitlab.com/user/project/pages/#parallel-deployments)\n- [GitLab Pages examples](https://gitlab.com/pages)",[9,495],{"slug":1143,"featured":6,"template":700},"build-a-new-website-in-a-few-easy-steps-with-gitlab-pages","content:en-us:blog:build-a-new-website-in-a-few-easy-steps-with-gitlab-pages.yml","Build A New Website In A Few Easy Steps With Gitlab Pages","en-us/blog/build-a-new-website-in-a-few-easy-steps-with-gitlab-pages.yml","en-us/blog/build-a-new-website-in-a-few-easy-steps-with-gitlab-pages",{"_path":1149,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1150,"content":1156,"config":1163,"_id":1165,"_type":13,"title":1166,"_source":15,"_file":1167,"_stem":1168,"_extension":18},"/en-us/blog/build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow",{"title":1151,"description":1152,"ogTitle":1151,"ogDescription":1152,"noIndex":6,"ogImage":1153,"ogUrl":1154,"ogSiteName":685,"ogType":686,"canonicalUrls":1154,"schema":1155},"Build an ML app pipeline with GitLab Model Registry using MLflow","Learn how to manage your ML apps entirely through GitLab with this tutorial. Also discover the role machine learning operations, or MLOps, plays in automating the DevSecOps lifecycle.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749660151/Blog/Hero%20Images/blog-image-template-1800x945__26_.png","https://about.gitlab.com/blog/build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Build an ML app pipeline with GitLab Model Registry using MLflow\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Gufran Yeşilyurt, OBSS\"},{\"@type\":\"Person\",\"name\":\"Péter Bozsó\"}],\n        \"datePublished\": \"2024-09-17\",\n      }",{"title":1151,"description":1152,"authors":1157,"heroImage":1153,"date":1160,"body":1161,"category":718,"tags":1162},[1158,1159],"Gufran Yeşilyurt, OBSS","Péter Bozsó","2024-09-17","__*Editor's note: From time to time, we invite members of our partner\ncommunity to contribute to the GitLab Blog. Thanks to Gufran Yeşilyurt, a\nDevOps consultant at OBSS Technology, for co-creating with us.*__\n\n\nThis tutorial will walk you through setting up an MLOps pipeline with GitLab\nModel Registry, utilizing MLflow. This will be a great starting point to\nmanage your ML apps entirely through GitLab. But first, it is crucial to\nunderstand why we need MLOps and what GitLab offers.\n\n\n[MLOps](https://about.gitlab.com/direction/modelops/mlops/#overview), or\nmachine learning operations, is a critical practice for managing and\nautomating the lifecycle of machine learning models, from development to\ndeployment and maintenance. Its importance lies in addressing the complexity\nand dynamism of machine learning workflows, which involve not just software\ndevelopment but also data management, model training, testing, deployment,\nand continuous monitoring.\n\n\nMLOps ensures that models are reproducible, scalable, and maintainable,\nfacilitating collaboration between data scientists, machine learning\nengineers, and operations teams. By incorporating MLOps, organizations can\nstreamline the deployment process, reduce time to market, and improve the\nreliability and performance of their machine learning applications.\n\n\nThe necessity of MLOps arises from the unique challenges posed by machine\nlearning projects. Unlike traditional software development, machine learning\ninvolves handling large datasets, experimenting with various models, and\ncontinuously updating models based on new data and feedback.\n\n\nWithout proper operations, managing these aspects becomes cumbersome,\nleading to potential issues like model drift, where the model's performance\ndegrades over time due to changes in the underlying data. MLOps provides a\nstructured approach to monitor and manage these changes, ensuring that\nmodels remain accurate and effective. Moreover, it introduces automation in\nvarious stages, such as data preprocessing, model training, and deployment,\nthereby reducing manual errors and enhancing efficiency.\n\n\nGitLab's features play a pivotal role in implementing MLOps effectively.\nGitLab provides an integrated platform that combines source code management,\n[CI/CD pipelines](https://about.gitlab.com/topics/ci-cd/), tracking and\ncollaboration tools, making it ideal for managing machine learning projects.\n\n\nWith GitLab, teams can leverage version control to track changes in both\ncode and data, ensuring reproducibility and transparency. The CI/CD\npipelines in GitLab automate the testing and deployment of machine learning\nmodels, allowing for continuous integration and continuous delivery. This\nautomation not only speeds up the deployment process but also ensures\nconsistency and reliability in the models being deployed. \n\n\nAdditionally, GitLab's collaboration features, such as merge requests and\ncode reviews, facilitate better communication and coordination among team\nmembers, ensuring that everyone is aligned and any issues are promptly\naddressed.\n\n\nPrerequisites:\n\n- basic knowledge of GitLab pipelines\n\n- basic knowledge of MLflow\n\n- a Kubernetes cluster\n\n- Dockerfile\n\n\nThis tutorial includes instructions to:\n\n- [Set up environment variables of\nMLflow](#set-up-environment-variables-of-mlflow)\n\n- [Train and log candidates at merge\nrequest](#train-and-log-candidates-at-merge-request)\n\n- [Register the most successful\ncandidate](#register-the-most-successful-candidate)\n\n- [Dockerize and deploy an ML app with the registered\nmodel](#dockerize-and-deploy-an-ml-app-with-the-registered-model)\n\n\nIn this example, to decide whether to provide the user a loan, we make use\nof Random Forest Classifier, Decision Tree, and Logistic Regression. At the\nend of this showcase, we will have a web application that utilizes machine\nlearning to respond to the user.\n\n\nTo reproduce this example in your own GitLab environment, you can read the\nrest of this article or follow the video below. You can find the source code\nof this example in [these OBSS\nrepositories](https://gitlab.com/gitlab-partners-public/obss).\n\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/grNJAp1xAi0?si=Bf9CAP9lB1uWErOZ\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Set up environment variables of MLflow\n\n\nOn the host where the code is executed, set the environment variables for\ntracking URI and token. This might be a remote host, CI pipeline, or your\nlocal environment. When they are set, you can call\n`mlflow.set_experiment(\"\u003Cexperiment_name>\")`. As a reference:\n\n\n```\n\nexport MLFLOW_TRACKING_URI=\"\u003Cyour gitlab endpoint>/api/v4/projects/\u003Cyour\nproject id>/ml/mlflow\"\n\nexport MLFLOW_TRACKING_TOKEN=\"\u003Cyour_access_token>\"\n\n```\n\n\n**Note:** If the training code contains the call to\n`mlflow.set_tracking_uri()`, remove it.\n\n\n## Train and log candidates at merge request\n\n\nIn your model train code, you can use MLflow methods to log metrics,\nartifacts, and parameters. You can also divide the train steps into pipeline\nstages if you are comfortable with that part. In this example, one Python\nfile will be used for both training and report generation.\n\n\n```\n\nmlflow.log_params(params)\n\nmlflow.log_metrics(metrics_data)\n\nmlflow.log_artifact(artifacts)\n\n```\n\n\nYou can then create the necessary pipeline to train the experiment. By\nadding the relevant rules, you can trigger this pipeline manually in merge\nrequests and observe the report generated as MR Note.\n\n\nWhen the pipeline is finished, you can see the details about the candidate\nin **Analyze > Model Experiments**.\n\n\n![details about the candidate in the finished\npipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676127/Blog/Content%20Images/Screenshot_1.png)\n\n\n## Register the most successful candidate\n\n\nAccording to the measurements you have made, we can register the most\nsuccessful candidate (may be the one with the highest accuracy value) with\nthe Run ID of the candidate.\n\n\nBut first, we need to create a model and its version in Registry. I created\nthese steps in separate stages and components (because I may need these\nsteps in other projects). You should be careful to use semantic versioning\nwhen versioning.\n\n\n### Register source model parameters and metrics\n\n\n```\n\nsource_candidate = client.get_run(source_candidate_id)\n\nparams = { k: v for k, v in source_candidate.data.params.items() }\n\nmetric = { k: v for k, v in source_candidate.data.metrics.items() }\n\n\nmodel_version = client.get_model_version(model_name, version)\n\nrun_id = model_version.run_id\n\nmodel_class = \"\"\n\nfor name, value in params.items():\n    client.log_param(run_id, name, value)\n    if name == \"Class\":\n        model_class = value\n\nfor name, value in metric.items():\n    client.log_metric(run_id, name, value)\n\n```\n\n\nAfter logging the parameters and metrics, you can [register the\nartifacts](https://gitlab.com/gitlab-partners-public/obss/mlops-loan-prediction/-/blob/main/register_candidate.py)\nas you did in the train step.\n\n\nYou may want to manually enter the inputs of the relevant steps as [a\nvariable in the\npipeline](https://gitlab.com/gitlab-partners-public/obss/components/-/blob/main/templates/register-candidate.yml).\n\n\n## CI/CD components\n\n\nI have used [CI/CD components](https://docs.gitlab.com/ee/ci/components/)\nbecause they provide a structured environment for managing machine learning\nworkflows. These components enable reusability by allowing teams to store\nand share standardized scripts, models, and datasets, ensuring that previous\nwork can be easily accessed, modified, and redeployed in future projects,\nthus accelerating development and reducing redundancy.\n\n\n> [Learn more about CI/CD components and the CI/CD\nCatalog](https://about.gitlab.com/blog/ci-cd-catalog-goes-ga-no-more-building-pipelines-from-scratch/).\n\n\n## Dockerize and deploy an ML app with the registered model\n\n\nIn this project, while registering the model, I also register the pkl file\nas an artifact and then create the docker image with that artifact and send\nit to [GitLab Container\nRegistry](https://about.gitlab.com/blog/next-generation-gitlab-container-registry-goes-ga/).\n\n\nYou can now access your Docker image from the Container Registry and deploy\nit to your environment with the method you want.\n\n\n## Resources\n\n- [Model\nexperiments](https://docs.gitlab.com/ee/user/project/ml/experiment_tracking/)\n\n- [MLflow client\ncompatibility](https://docs.gitlab.com/ee/user/project/ml/experiment_tracking/mlflow_client.html)\n\n- [CI/CD components](https://docs.gitlab.com/ee/ci/components/)\n\n- [Building GitLab with GitLab: Why there is no MLOps without\nDevSecOps](https://about.gitlab.com/blog/there-is-no-mlops-without-devsecops/)\n\n\n***Credits:**\n\nThis tutorial and the corresponding sample projects were created and\ngenerously shared with the community by [OBSS](https://obss.tech/en/). OBSS\nis an EMEA-based channel partner of GitLab. They have deep expertise across\nthe whole DevSecOps lifecycle and amongst many other things, they are more\nthan happy to support customers with migrating their MLOps workloads to\nGitLab.*\n",[720,9,108,283],{"slug":1164,"featured":90,"template":700},"build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow","content:en-us:blog:build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow.yml","Build An Ml App Pipeline With Gitlab Model Registry Using Mlflow","en-us/blog/build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow.yml","en-us/blog/build-an-ml-app-pipeline-with-gitlab-model-registry-using-mlflow",{"_path":1170,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1171,"content":1177,"config":1182,"_id":1184,"_type":13,"title":1185,"_source":15,"_file":1186,"_stem":1187,"_extension":18},"/en-us/blog/build-and-run-containers-in-remote-development-workspaces",{"title":1172,"description":1173,"ogTitle":1172,"ogDescription":1173,"noIndex":6,"ogImage":1174,"ogUrl":1175,"ogSiteName":685,"ogType":686,"canonicalUrls":1175,"schema":1176},"Build and run containers in Remote Development workspaces","Use this easy-to-follow tutorial to create a secure, ephemeral, reproducible development environment in GitLab that can replace your local environments.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663857/Blog/Hero%20Images/blog-image-template-1800x945__12_.png","https://about.gitlab.com/blog/build-and-run-containers-in-remote-development-workspaces","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Build and run containers in Remote Development workspaces\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Vishal Tak\"}],\n        \"datePublished\": \"2025-03-03\",\n      }",{"title":1172,"description":1173,"authors":1178,"heroImage":1174,"date":1139,"body":1180,"category":694,"tags":1181},[1179],"Vishal Tak","Development environments often require the ability to build and run\ncontainers as part of their local development. Securely running containers\nwithin containers can be challenging. This article will provide a\nstep-by-step guide to securely build and run containers in a workspace.\n\n\nYou will learn how to:\n\n- [Create a Kubernetes cluster on AWS\nEKS](#create-a-kubernetes-cluster-on-aws-eks)\n\n- [Configure Sysbox](#configure-sysbox)\n\n- [Configure GitLab agent for Kubernetes and GitLab Workspaces\nProxy](#configure-gitlab-agent-for-kubernetes-and-gitlab-workspaces-proxy)\n\n- [Configure sudo access for a workspace with\nSysbox](#configure-sudo-access-for-a-workspace-with-sysbox)\n\n- [Configure Ingress Controller](#configure-ingress-controller)\n\n- [Build containers inside a\nworkspace](#build-containers-inside-a-workspace)\n\n- [Run containers inside a workspace](#run-containers-inside-a-workspace)\n\n- [Get started today](#get-started-today)\n\n\n## Create a Kubernetes cluster on AWS EKS\n\nInstall the [AWS\nCLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)\non your local machine. Next, configure a [named\nprofile](https://docs.aws.amazon.com/cli/latest/reference/configure/) and\nexport it to ensure all the following `aws` commands use the set\ncredentials.\n\n\n```\n\naws configure --profile gitlab-workspaces-container-demo\n\nexport AWS_PROFILE=gitlab-workspaces-container-demo\n\n```\n\n\nInstall [eksctl](https://eksctl.io/installation/), a CLI to interact with\nAWS EKS. Let’s now create a Kubernetes 1.31 cluster on AWS EKS with 1 node\nof Ubuntu 22.04 of `c5.2xlarge` instance type. The nodes can autoscale from\n0-20 nodes and each node will have a label `sysbox-install: yes` . This will\nbe explained later in the article.\n\n\n```\n\nexport CLUSTER_NAME=\"gitlab-workspaces-container-demo-eks-sysbox\"\n\n\neksctl create cluster \\\n  --name \"${CLUSTER_NAME}\" \\\n  --version 1.31 \\\n  --node-ami-family=Ubuntu2204 \\\n  --nodes=1 \\\n  --nodes-min=0 \\\n  --nodes-max=20 \\\n  --instance-types=c5.2xlarge \\\n  --node-labels \"sysbox-install=yes\" \\\n  --asg-access \\\n  --external-dns-access \\\n  --full-ecr-access\n```\n\n\nCreate an [IAM\nOIDC](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html)\nprovider for your cluster.\n\n\n```\n\neksctl utils associate-iam-oidc-provider --cluster \"${CLUSTER_NAME}\"\n--approve\n\n```\n\n\nCreate IAM role for [EBS\nadd-on](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) for\nEKS.\n\n\n```\n\neksctl create iamserviceaccount \\\n  --name ebs-csi-controller-sa \\\n  --namespace kube-system \\\n  --cluster \"${CLUSTER_NAME}\" \\\n  --role-name \"AmazonEKS_EBS_CSI_DriverRole_${CLUSTER_NAME}\" \\\n  --role-only \\\n  --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \\\n  --approve\n```\n\n\nCreate Amazon EBS CSI driver add-on for Amazon EKS cluster.  \n\n\n```\n\neksctl utils describe-addon-versions --kubernetes-version 1.31 | grep\naws-ebs-csi-driver\n\n\nexport AWS_ACCOUNT_ID=\"UPDATE_ME\"\n\n\neksctl create addon \\\n  --cluster \"${CLUSTER_NAME}\" \\\n  --name aws-ebs-csi-driver \\\n  --version latest \\\n  --service-account-role-arn \"arn:aws:iam::${AWS_ACCOUNT_ID}:role/AmazonEKS_EBS_CSI_DriverRole_${CLUSTER_NAME}\" \\\n  --force\n```\n\n\nInstall [kubectl](https://kubernetes.io/docs/reference/kubectl/), a command\nline tool for communicating with a Kubernetes cluster's control plane, using\nthe Kubernetes API.\n\n\nLet’s get the\n[kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)\nof the created cluster.\n\n\n```\n\naws eks update-kubeconfig --name \"${CLUSTER_NAME}\"\n\n```\n\n\n## Configure Sysbox\n\n\n[Sysbox](https://github.com/nestybox/sysbox) is a container runtime that\nimproves container isolation and enables containers to run the same\nworkloads as virtual machines.\n\n\n[Install](https://github.com/nestybox/sysbox#installation) Sysbox on the\nKubernetes cluster using the `sysbox-deploy-k8s daemonset`.\n\n\n```\n\ncurl\nhttps://raw.githubusercontent.com/nestybox/sysbox/refs/tags/v0.6.6/sysbox-k8s-manifests/sysbox-install.yaml\n-o sysbox-install.yaml\n\n```\n\n\nBecause of how Sysbox releases itself, it first created a git tag, which\nruns a pipeline to build assets after which the YAML files for the\n`sysbox-deploy-k8s daemonset` are updated. Thus, we need to update the\nDaemonSet's `spec.template.soec.containers[0].image` to\n[registry.nestybox.com/nestybox/sysbox-deploy-k8s:v0.6.6-0](https://github.com/nestybox/sysbox/blob/46ba726e8e894aa22e20465a32d22dfa2863ec12/sysbox-k8s-manifests/sysbox-install.yaml#L66)\n.\n\n\n```\n\nnew_image_value=\"registry.nestybox.com/nestybox/sysbox-deploy-k8s:v0.6.6-0\"\n\ntemp_file=$(mktemp)\n\nsed -E \"s|^([[:space:]]*image:)[[:space:]]*.*|\\1 $new_image_value|\"\n\"sysbox-install.yaml\" > \"$temp_file\"\n\nmv \"$temp_file\" \"sysbox-install.yaml\"\n\n```\n\n\nApply the YAML file to Kubernetes and ensure all the pods of the DaemonSet\nare running.\n\n\n```\n\nkubectl apply -f sysbox-install.yaml\n\nkubectl get pod -A\n\nkubectl -n kube-system get daemonset\n\n```\n\n\nVerify the installation by creating a pod which uses Sysbox container\nruntime.\n\n\n```\n\ncat \u003C\u003CEOF | kubectl apply -f -\n\napiVersion: v1\n\nkind: Pod\n\nmetadata:\n  name: sysbox-verification-pod\n  namespace: default\n  annotations:\n    io.kubernetes.cri-o.userns-mode: \"auto:size=65536\"\nspec:\n  runtimeClassName: sysbox-runc\n  containers:\n  - image: \"hello-world\"\n    imagePullPolicy: Always\n    name: main\n  restartPolicy: Always\nEOF\n\n\nkubectl -n default get pod sysbox-verification-pod\n\nkubectl exec -it sysbox-verification-pod -- echo \"Pod is running\nsuccessfully on a Kubernetes cluster configured with Sysbox.\"\n\nkubectl -n default delete pod sysbox-verification-pod\n\n```\n\n\n## Configure GitLab agent for Kubernetes and GitLab Workspaces Proxy\n\n\nFollow our [documentation\ntutorial](https://docs.gitlab.com/ee/user/workspace/set_up_gitlab_agent_and_proxies.html)\nto set up GitLab agent and GitLab Workspaces Proxy.  \n\n\n## Configure sudo access for a workspace with Sysbox\n\n\nFollow our\n[documentation](https://docs.gitlab.com/ee/user/workspace/configuration.html#with-sysbox)\nto configure sudo access for a workspace with Sysbox.\n\n\n## Configure Ingress Controller\n\n\nSetup [Ingress NGINX Controller for\nKubernetes](https://github.com/kubernetes/ingress-nginx)\n\n\n```\n\nhelm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx\n--force-update\n\nhelm repo update\n\n\nhelm upgrade --install \\\n  ingress-nginx ingress-nginx/ingress-nginx \\\n  --namespace ingress-nginx \\\n  --create-namespace \\\n  --version 4.11.1 \\\n  --timeout=600s --wait --wait-for-jobs\n\nkubectl -n ingress-nginx get pod\n\n```\n\n\n## Build containers inside a workspace\n\n\nWe’ll use\n[example-go-http-app](https://gitlab.com/gitlab-org/workspaces/examples/example-go-http-app)\nas the project to create a workspace from. Open the workspace, start a\nterminal, and install [Docker](https://docs.docker.com/engine/install/).\n\n\n```\n\n# Add Docker's official GPG key:\n\nsudo apt-get update\n\nsudo apt-get install ca-certificates curl\n\nsudo install -m 0755 -d /etc/apt/keyrings\n\nsudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o\n/etc/apt/keyrings/docker.asc\n\nsudo chmod a+r /etc/apt/keyrings/docker.asc\n\n\n# Add the repository to Apt sources:\n\necho \\\n  \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \\\n  $(. /etc/os-release && echo \"${UBUNTU_CODENAME:-$VERSION_CODENAME}\") stable\" | \\\n  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null\nsudo apt-get update\n\nsudo apt-get install -y docker-ce docker-ce-cli containerd.io\ndocker-buildx-plugin docker-compose-plugin\n\n\n# Start the Docker Daemon\n\nsudo dockerd\n\n```\n\n\nBuild the container image.\n\n\n```\n\nsudo docker build -t workspaces-golang-server .\n\n```\n\n\n## Run containers inside a workspace\n\n\nLet’s run the container built above and expose port 3000 from the container\nonto the host (workspace).\n\n\n```\n\nsudo docker run -p 3000:3000 workspaces-golang-server\n\n```\n\n\nThe port `3000` is exposed in the\n[.devfile.yaml](https://gitlab.com/gitlab-org/workspaces/examples/example-go-http-app/-/blob/dd3dbb38cdce1143f7ed023980f34630cea991a5/.devfile.yaml#L15)\nused to create the workspace. Access the server running inside the container\nfrom the browser. Here is a video clip.\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/JQErF0U6oFk?si=6oiK48q5ghZq312g\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Get started today\n\n\nFrom GitLab 17.4, you can build and run containers securely in GitLab\nWorkspaces. See our\n[documentation](https://docs.gitlab.com/ee/user/workspace/configuration.html#build-and-run-containers-in-a-workspace)\nfor more information. Replace your local development environments to GitLab\nWorkspaces for a secure, ephemeral, reproducible development environment. \n\n\n## Read more\n\n\n- [Enable secure sudo access for GitLab Remote Development\nworkspaces](https://about.gitlab.com/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces/)\n\n- [Quickstart guide for GitLab Remote Development\nworkspaces](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/)\n\n- [Create a workspace quickly with the GitLab default\ndevfile](https://about.gitlab.com/blog/create-a-workspace-quickly-with-the-gitlab-default-devfile/)\n\n- [Contributor how-to: Remote Development workspaces and GitLab Developer\nKit](https://about.gitlab.com/blog/gitlab-gdk-remote-development/)\n",[9,495,696,694],{"slug":1183,"featured":90,"template":700},"build-and-run-containers-in-remote-development-workspaces","content:en-us:blog:build-and-run-containers-in-remote-development-workspaces.yml","Build And Run Containers In Remote Development Workspaces","en-us/blog/build-and-run-containers-in-remote-development-workspaces.yml","en-us/blog/build-and-run-containers-in-remote-development-workspaces",{"_path":1189,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1190,"content":1195,"config":1200,"_id":1202,"_type":13,"title":1203,"_source":15,"_file":1204,"_stem":1205,"_extension":18},"/en-us/blog/building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way",{"title":1191,"description":1192,"ogTitle":1191,"ogDescription":1192,"noIndex":6,"ogImage":1153,"ogUrl":1193,"ogSiteName":685,"ogType":686,"canonicalUrls":1193,"schema":1194},"Building a GitLab CI/CD pipeline for a monorepo the easy way","Learn how to create a GitLab CI/CD pipeline for a monorepo to host multiple applications in one repository.","https://about.gitlab.com/blog/building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Building a GitLab CI/CD pipeline for a monorepo the easy way\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sam Morris\"}],\n        \"datePublished\": \"2024-07-30\",\n      }",{"title":1191,"description":1192,"authors":1196,"heroImage":1153,"date":1197,"body":1198,"category":785,"tags":1199},[846],"2024-07-30","Monorepos allow you to host multiple applications’ code in a single\nrepository. In GitLab, that involves placing disparate application source\ncode in separate directories in one project. While this strategy allows for\nversion controlled storage of your code, it was tricky leveraging the full\npower of GitLab’s [CI/CD](https://about.gitlab.com/topics/ci-cd/) pipeline\ncapabilities… until now!\n\n\n## The ideal case: CI/CD in a monorepo\n\n\nSince you have more than one application’s code living in your repository,\nyou will want to have more than one pipeline configuration. For example, if\nyou have a .NET application and a Spring application in one project, each\napplication may have different build and test jobs to complete. Ideally, you\ncan completely decouple the pipelines and only run each pipeline based on\nchanges to that specific application’s source code.\n\n\nThe technical approach for this would be to have a project-level\n`.gitlab-ci.yml` pipeline configuration file that includes a specific YAML\nfile based on changes in a certain directory. The `.gitlab-ci.yml` pipeline\nserves as the control plane that triggers the appropriate pipeline based on\nthe changes made to the code.\n\n\n## The legacy approach\n\n\nPrior to GitLab 16.4, we were not able to include a YAML file based on\nchanges to a directory or file in a project. However, we could accomplish\nthis functionality via a workaround. \n\n\nIn our monorepo project, we have two directories for different applications.\nIn this example, there are `java` and `python` directories representing a\nJava and Python app, respectively. Each directory has an\napplication-specific YAML file to build each app. In the project’s pipeline\nfile, we simply include both application pipeline files, and do the logic\nhandling in those files directly.\n\n\n`.gitlab-ci.yml`:\n\n\n```\n\nstages:\n  - build\n  - test\n  - deploy\n\ntop-level-job:\n  stage: build\n  script:\n    - echo \"Hello world...\"\n\ninclude:\n  - local: '/java/j.gitlab-ci.yml'\n  - local: '/python/py.gitlab-ci.yml'\n\n```\n\n\nIn each application-specific pipeline file, we create a hidden job named\n.java-common or .python-common that only runs if there are changes to that\napp’s directory. [Hidden\njobs](https://docs.gitlab.com/ee/ci/jobs/#hide-jobs) do not run by default,\nand are often utilized to reuse specific job configurations. Each pipeline\nextends that hidden job to inherit the rules defining which files to watch\nfor changes, which would then initiate the pipeline job. \n\n\n`j.gitlab-ci.yml`:\n\n\n```\n\nstages:\n  - build\n  - test\n  - deploy\n\n.java-common:\n  rules:\n    - changes:\n      - '../java/*'\n\njava-build-job:\n  extends: .java-common\n  stage: build\n  script:\n    - echo \"Building Java\"\n\njava-test-job:\n  extends: .java-common\n  stage: test\n  script:\n    - echo \"Testing Java\"\n\n```\n\n\n`py.gitlab-ci.yml`:\n\n\n```\n\nstages:\n  - build\n  - test\n  - deploy\n\n.python-common:\n  rules:\n    - changes:\n      - '../python/*'\n\npython-build-job:\n  extends: .python-common\n  stage: build\n  script:\n    - echo \"Building Python\"\n\npython-test-job:\n  extends: .python-common\n  stage: test\n  script:\n    - echo \"Testing Python\"\n\n```\n\n\nThere are some downsides to this, including having to extend the job for\neach other job in the YAML file to ensure it complies with the rules,\ncreating a lot of redundant code and room for human error. Additionally,\nextended jobs cannot have duplicate keys, so you could not define your own\n`rules` logic in each job since there would be a collision in the keys and\ntheir [values are not\nmerged](https://docs.gitlab.com/ee/ci/yaml/index.html#extends). \n\n\nThis results in a pipeline running that includes the j.gitlab-ci.yml jobs\nwhen `java/` is updated, and py.gitlab-ci.yml when `python/` is updated. \n\n\n## The new approach: Conditionally include pipeline files\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/6phvk8jioAo?si=y6ztZODvUtM-cHmZ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nIn GitLab 16.4, we introduced [`include` with `rules:changes` for\npipelines](https://docs.gitlab.com/ee/ci/yaml/includes.html#include-with-ruleschanges).\nPreviously, you could `include` with `rules:if`, but not `rules:changes`\nmaking this update extremely powerful. Now, you can simply use the `include`\nkeyword and define the monorepo rules in your project pipeline\nconfiguration. \n\n\nNew `.gitlab-ci.yml`:\n\n\n```\n\nstages:\n  - build\n  - test\n\ntop-level-job:\n  stage: build\n  script:\n    - echo \"Hello world...\"\n\ninclude:\n  - local: '/java/j.gitlab-ci.yml'\n    rules:\n      - changes:\n        - 'java/*'\n  - local: '/python/py.gitlab-ci.yml'\n    rules:\n      - changes:\n        - 'python/*'\n\n```\n\n\nThen each application’s YAML can just focus on building and testing that\napplication’s code, without extending a hidden job repeatedly. This allows\nfor more flexibility in job definitions and reduces code rewriting for\nengineers.\n\n\nNew `j.gitlab-ci.yml`:\n\n\n```\n\nstages:\n  - build\n  - test\n  - deploy\n\njava-build-job:\n  stage: build\n  script:\n    - echo \"Building Java\"\n\njava-test-job:\n  stage: test\n  script:\n    - echo \"Testing Java\"\n\n```\n\n\nNew `py.gitlab-ci.yml`:\n\n```\n\nstages:\n  - build\n  - test\n  - deploy\n\npython-build-job:\n  stage: build\n  script:\n    - echo \"Building Python\"\n\npython-test-job:\n  stage: test\n  script:\n    - echo \"Testing Python\"\n\n```\n\n\nThis accomplishes the same task of including the Java and Python jobs only\nwhen their directories are modified. Something to consider in your\nimplementation is that [jobs can run unexpectedly when using\n`changes`](https://docs.gitlab.com/ee/ci/jobs/job_troubleshooting.html#jobs-or-pipelines-run-unexpectedly-when-using-changes).\nThe changes rule always evaluates to true when pushing a new branch or a new\ntag to GitLab, so all jobs included will run upon first push to a branch\nregardless of the `rules:changes` definition. You can mitigate this\nexperience by creating your feature branch first and then opening a merge\nrequest to begin your development, since the first push to the branch when\nit is created will force all jobs to run.\n\n\nUltimately, monorepos are a strategy that can be used with GitLab and CI/CD,\nand, with our new `include` with `rules:changes` feature, we have a better\nbest practice for using GitLab CI with monorepos. To get started with\nmonorepos, take out a free Gitlab Ultimate trial today.\n\n\n## More CI/CD resources\n\n\n* [5 tips for managing monorepos in\nGitLab](https://about.gitlab.com/blog/tips-for-managing-monorepos-in-gitlab/)\n\n* [How to learn CI/CD\nfast](https://about.gitlab.com/blog/how-to-learn-ci-cd-fast/)\n",[108,9],{"slug":1201,"featured":6,"template":700},"building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way","content:en-us:blog:building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way.yml","Building A Gitlab Ci Cd Pipeline For A Monorepo The Easy Way","en-us/blog/building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way.yml","en-us/blog/building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way",{"_path":1207,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1208,"content":1214,"config":1221,"_id":1223,"_type":13,"title":1224,"_source":15,"_file":1225,"_stem":1226,"_extension":18},"/en-us/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions",{"title":1209,"description":1210,"ogTitle":1209,"ogDescription":1210,"noIndex":6,"ogImage":1211,"ogUrl":1212,"ogSiteName":685,"ogType":686,"canonicalUrls":1212,"schema":1213},"Explore the Dragon Realm: Building a C++ adventure game with AI","How to use GitLab Duo Code Suggestions to create a text-based adventure game, including magical locations to visit and items to procure, using C++.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663344/Blog/Hero%20Images/compassinfield.jpg","https://about.gitlab.com/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Explore the Dragon Realm: Build a C++ adventure game with a little help from AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fatima Sarah Khalid\"}],\n        \"datePublished\": \"2023-08-24\",\n      }",{"title":1215,"description":1210,"authors":1216,"heroImage":1211,"date":1218,"body":1219,"category":718,"tags":1220},"Explore the Dragon Realm: Build a C++ adventure game with a little help from AI",[1217],"Fatima Sarah Khalid","2023-08-24","Learning, for me, has never been about reading a textbook or sitting in on a\nlecture - it's been about experiencing and immersing myself in a hands-on\nchallenge. This is particulary true for new programming languages. With\n[GitLab Duo Code Suggestions](https://about.gitlab.com/gitlab-duo/),\nartificial intelligence (AI) becomes my interactive guide, providing an\nenvironment for trial, error, and growth. In this tutorial, we will build a\ntext-based adventure game in C++ by using Code Suggestions to learn the\nprogramming language along the way.\n\n\nYou can use this table of contents to navigate into each section. It is\nrecommended to read top-down for the best learning experience.\n\n\n- [Setup](#setup)\n  - [Installing VS Code](#installing-vs-code)\n  - [Installing Clang as a compiler](#installing-clang-as-a-compiler)\n  - [Setting up VS Code](#setting-up-vs-code)\n- [Getting started](#getting-started)\n  - [Compiling and running your program](#compiling-and-running-your-program)\n- [Setting the text adventure stage](#setting-the-adventure-stage)\n\n- [Defining the adventure: Variables](#defining-the-adventure-variables)\n\n- [Crafting the adventure: Making decisions with\nconditionals](#crafting-the-adventure-making-decisions-with-conditionals)\n\n- [Structuring the narrative:\nCharacters](#structuring-the-narrative-characters)\n\n- [Structuring the narrative: Items](#structuring-the-narrative-items)\n\n- [Applying what we've learned at the Grand\nLibrary](#applying-what-weve-learned-at-the-grand-library)\n\n- [See you next time in the Dragon\nRealm](#see-you-next-time-in-the-dragon-realm)\n\n- [Share your feedback](#share-your-feedback)\n\n\n> Download [GitLab Ultimate for free](https://about.gitlab.com/gitlab-duo/)\nfor a trial of GitLab Duo Code Suggestions.\n\n\n## Setup\n\nYou can follow this tutorial in your [preferred and supported\nIDE](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-in-other-ides-and-editors).\nReview the documentation to enable Code Suggestions for [GitLab.com\nSaaS](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-gitlab-saas)\nor [GitLab self-managed\ninstances](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-self-managed-gitlab).\n\n\nThese installation instructions are for macOS Ventura on M1 Silicon. \n\n\n### Installing VS Code\n\n\n* Download and install [VS Code](https://code.visualstudio.com/download).\n\n* Alternatively, you can also install it as a Homebrew cask: `brew install\n--cask visual-studio-code`.\n\n\n### Installing Clang as a compiler\n\n\n* On macOS, you'll need to install some developer tools. Open your terminal\nand type:\n\n\n```\n\nxcode-select --install\n\n```\n\n\nThis will prompt you to install Xcode's command line tools, which include\nthe [Clang C++ compiler](https://clang.llvm.org/get_started.html).\n\n\nAfter the installation, you can check if `clang++` is installed by typing:\n\n\n```\n\nclang++ --version\n\n```\n\n\nYou should see an output that includes some information about the Clang\nversion you have installed. \n\n\n### Setting up VS Code\n\n\n* Launch VS Code.\n\n* Install and configure [the GitLab Workflow\nextension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow).\n\n* Optionally, in VS Code, install the [C/C++ Intellisense\nextension](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools),\nwhich helps with debugging C/C++. \n\n\n## Getting started\n\nNow, let's start building this magical adventure with C++. We'll start with\na \"Hello World\" example.\n\n\nCreate a new project `learn-ai-cpp-adventure`. In the project root, create\n`adventure.cpp`. The first part of every C++ program is the `main()`\nfunction. It's the entry point of the program.\n\n\nWhen you start writing `int main() {`, Code Suggestions will help\nautocomplete the function with some default parameters.\n\n\n![adventure.cpp with a hello world implementation suggested by Code\nSuggestions](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/0-helloworld.png){:\n.shadow}\n\n\n```cpp\n\nint main()\n\n{\n    cout \u003C\u003C \"Hello World\" \u003C\u003C endl;\n    return 0;\n}\n\n```\n\n\nWhile this is a good place to start, we need to add an include and update\nthe output statement:\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    // Print \"Hello World!\" to the console\n    std::cout \u003C\u003C \"Hello World!\" \u003C\u003C std::endl;\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\nThe program prints \"Hello World!\" to the console when executed.\n\n\n* `#include \u003Ciostream>`: Because we are building a text-based adventure, we\nwill rely on input from the player using input and output operations (I/O)\nin C++. This include is a preprocessor directive that tells our program to\ninclude the `iostream` library, which provides facilities to use input and\noutput streams, such as `std::cout` for output.\n\n\n* You might find that Code Suggestions suggests `int main(int argc, char*\nargv[])` as the definition of our main function. The parameters `(int argc,\nchar* argv[])` are used to pass command-line arguments to the program. Code\nSuggestions added them as default parameters, but they are not needed if\nyou're not using command-line arguments. In that case, we can also define\nthe main function as `int main()`.\n\n\n* `std::cout \u003C\u003C \"Hello World!\" \u003C\u003C std::endl;`: outputs \"Hello World\" to the\nconsole. The stream operator `\u003C\u003C` is used to send the string to output.\n`std::endl` is an end-line character.\n\n\n* `return 0;`: we use `return 0;` to indicate the end of the `main()`\nfunction and return a value of 0. In C++, it is good practice to return 0 to\nindicate the program has completed successfully.\n\n\n### Compiling and running your program\n\nNow that we have some code, let's review how we'll compile and run this\nprogram. \n\n* Open your terminal or use the terminal in VSCode (View -> Terminal).\n\n* Navigate to your project directory.\n\n* Compile your program by typing:\n\n\n```bash\n\nclang++ adventure.cpp -o adventure\n\n```\n\n\nThis command tells the Clang++ compiler to compile adventure.cpp and create\nan executable named adventure. After this, run your program by typing:\n\n\n```\n\n./adventure\n\n```\n\n\nYou should see \"Hello World!\" printed in the terminal. \n\n\nBecause our tutorial uses a single source file `adventure.cpp`, we can use\nthe compiler directly to build our program. In the future, if the program\ngrows beyond a file, we'll set up additional configurations to handle\ncompilation. \n\n\n## Setting the text adventure stage\n\nBefore we get into more code, let's set the stage for our text adventure.\n\n\nFor this text adventure, players will explore the Dragon Realm. The Dragon\nRealm is full of mountains, lakes, and magic. Our player will enter the\nDragon Realm for the first time, explore different locations, meet new\ncharacters, collect magical items, and journal their adventure. At every\nlocation, they will be offered choices to decide the course of their\njourney.\n\n\nTo kick off our adventure into the Dragon Realm, let's update our\n`adventure.cpp main()` function to be more specific. As you update the\nwelcome message, you might find that Code Suggestions already knows we're\nbuilding a game.\n\n\n![adventure.cpp - Code Suggestions offers suggestion of welcoming users to\nthe Dragon Realm and knows its a\ngame](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/1-welcome-to-the-realm.png){:\n.shadow}\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    // Print \"Hello World!\" to the console\n    std::cout \u003C\u003C \"Welcome to the Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\n## Defining the adventure: Variables\n\nA variable stores data that can be used throughout the program scope in the\n`main()` function. A variable is defined by a type, which indicates the kind\nof data it can hold.\n\n\nLet's create a variable to hold our player's name and give it the type\n`string`. A `string` is designed to hold a sequence of characters so it's\nperfect for storing our player's name.\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    // Print \"Hello World!\" to the console\n    std::cout \u003C\u003C \"Welcome to the Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Declare a string variable to hold the player's name\n    std::string playerName;\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\nAs you do this, you may notice that Code Suggestions knows what's coming\nnext - prompting the user for their player's name.\n\n\n![adventure.cpp - Code Suggestions suggests welcoming the player with the\nplayerName\nvariable](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/2-player-name-variable.png){:\n.shadow}\n\n\nWe may be able to get more complete and specific Code Suggestions by\nproviding comments about what we'd like to do with the name - personally\nwelcome the player to the game. Start by adding our plan of action in\ncomments.\n\n\n```cpp\n    // Declare a string variable to hold the player's name\n    std::string playerName;\n\n    // Prompt the user to enter their player name\n\n    // Display a personalized welcome message to the player with their name\n```\n\n\nTo capture the player's name from input, we need to use the `std::cin`\nobject from the `iostream` library to fetch input from the player using the\nextraction operator `>>`. If you start typing `std::` to start prompting the\nuser, Code Suggestions will make some suggestions to help you gather user\ninput and save it to our `playerName` variable.\n\n\n![adventure.cpp - Code Suggestions prompts the user to input their player\nname](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/2.1-player-name-input.png){:\n.shadow}\n\n\nNext, to welcome our player personally to the game, we want to use\n`std::cout` and the `playerName` variable together:\n\n\n```cpp\n    // Declare a string variable to store the player name\n    std::string playerName;\n\n    // Prompt the user to enter their player name\n    std::cout \u003C\u003C \"Please enter your name: \";\n    std::cin >> playerName;\n\n    // Display a personalized welcome message to the player with their name\n    std::cout \u003C\u003C \"Welcome \" \u003C\u003C playerName \u003C\u003C \" to The Dragon Realm!\" \u003C\u003C std::endl;\n```\n\n\n## Crafting the adventure: Making decisions with conditionals\n\nIt's time to introduce our player to the different locations in tbe Dragon\nRealm they can visit. To prompt our player with choices, we use\nconditionals. Conditionals allow programs to take different actions based on\ncriteria, such as user input.\n\n\nLet's offer the player a selection of locations to visit and capture their\nchoice as an `int` value that corresponds to the location they picked.\n\n\n```cpp\n\n// Display a personalized welcome message to the player with their name\n\nstd::cout \u003C\u003C \"Welcome \" \u003C\u003C playerName \u003C\u003C \" to The Dragon Realm!\" \u003C\u003C\nstd::endl;\n\n\n// Declare an int variable to capture the user's choice\n\nint choice;\n\n```\n\n\nThen, we want to offer the player the different locations that are possible\nfor that choice. Let's start with a comment and prompt Code Suggestions with\n`std::cout` to fill out the details for us.\n\n\n![adventure.cpp - Code Suggestions suggests a multiline output for all the\nlocations listed in the code\nbelow](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3-setup-location-choice.png){:\n.shadow}\n\n\nAs you accept the suggestions, Code Suggestions will help build out the\noutput and ask the player for their input.\n\n\n![adventure.cpp - Code Suggestions suggests a multiline output for all the\nlocations listed in the code below and asks for player\ninput](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.1-capture-player-location-choice.png){:\n.shadow}\n\n\n```cpp\n    // Declare an int variable to capture the user's choice\n    int choice;\n\n    // Offer the player a choice of 3 locations: 1 for Moonlight Markets, 2 for Grand Library, and 3 for Shimmer Lake.\n    std::cout \u003C\u003C \"Where will \" \u003C\u003C playerName \u003C\u003C \" go?\" \u003C\u003C std::endl;\n    std::cout \u003C\u003C \"1. Moonlight Markets\" \u003C\u003C std::endl;\n    std::cout \u003C\u003C \"2. Grand Library\" \u003C\u003C std::endl;\n    std::cout \u003C\u003C \"3. Shimmer Lake\" \u003C\u003C std::endl;\n    std::cout \u003C\u003C \"Please enter your choice: \";\n    std::cin >> choice;\n```\n\n\nOnce you start typing `std::cin >>` or accept the prompt for asking the\nplayer for their choice, Code Suggestions might offer a suggestion for\nbuilding out your conditional flow. AI is non-deterministic: One suggestion\ncan involve if/else statements while another solution uses a switch\nstatement.\n\n\nTo give Code Suggestions a nudge, we'll add a comment and start typing out\nan if statement: `if (choice ==)`.\n\n\n![adventure.cpp - Code Suggestions suggests using an if statement to manage\nchoice of\nlocations](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.2-if-statement-locations.png){:\n.shadow}\n\n\nAnd if you keep accepting the subsequent suggestions, Code Suggestions will\nautocomplete the code using if/else statements.\n\n\n![adventure.cpp - Code Suggestions helps the user fill out the rest of the\nif/else statements for choosing a\nlocation](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.2.1-if-statement-locations-continued.png){:\n.shadow}\n\n\n```cpp\n    // Check the user's choice and display the corresponding messages\n    if (choice == 1) {\n        std::cout \u003C\u003C \"You chose Moonlight Markets\" \u003C\u003C std::endl;\n    }\n    else if (choice == 2) {\n        std::cout \u003C\u003C \"You chose Grand Library\" \u003C\u003C std::endl;\n    }\n    else if (choice == 3) {\n        std::cout \u003C\u003C \"You chose Shimmer Lake\" \u003C\u003C std::endl;\n    }\n    else {\n        std::cout \u003C\u003C \"Invalid choice\" \u003C\u003C std::endl;\n    }\n```\n\n\n`if/else` is a conditional statement that allows a program to execute code\nbased on whether a condition, in this case the player's choice, is true or\nfalse. If the condition evaluates to true, the code inside the braces is\nexecuted.\n\n\n* `if (condition)`: used to check if the condition is true.\n\n* `else if (another condition)`: if the previous condition isn't true, the\nprograms checks this condition.\n\n* `else`: if none of the previous conditions are true.\n\n\nAnother way of managing multiple choices like this example is using a\n`switch()` statement. A `switch` statement allows our program to jump to\ndifferent sections of code based on the value of an expression, which, in\nthis case, is the value of `choice`.\n\n\nWe are going to replace our `if/else` statements with a `switch` statement.\nYou can comment out or delete the `if/else` statements and prompt Code\nSuggestions starting with `switch(choice) {`.\n\n\n![adventure.cpp - Code Suggestions helps the user handle the switch\nstatement for the\nlocations](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.3-conditional-switch-locations.png){:\n.shadow}\n\n\n![adventure.cpp - Code Suggestions helps the user handle the switch\nstatement for the\nlocations](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.3.1-conditional-switch-locations-continued.png){:\n.shadow}\n\n\n```cpp\n    // Evaluate the player's decision\n    switch(choice) {\n        // If 'choice' is 1, this block is executed.\n        case 1:\n            std::cout \u003C\u003C \"You chose Moonlight Markets.\" \u003C\u003C std::endl;\n            break;\n        // If 'choice' is 2, this block is executed.\n        case 2:\n            std::cout \u003C\u003C \"You chose Grand Library.\" \u003C\u003C std::endl;\n            break;\n        // If 'choice' is 3, this block is executed.\n        case 3:\n            std::cout \u003C\u003C \"You chose Shimmer Lake.\" \u003C\u003C std::endl;\n            break;\n        // If 'choice' is not 1, 2, or 3, this block is executed.\n        default:\n            std::cout \u003C\u003C \"You did not enter 1, 2, or 3.\" \u003C\u003C std::endl;\n    }\n```\n\n\nEach case represents a potential value that the variable or expression being\nswitched on (in this case, choice) could have. If a match is found, the code\nfor that case is executed. We use the `default` case to handle any input\nerrors in case the player enters a value that isn't accounted for.\n\n\nLet's build out what happens when our player visits the Shimmering Lake.\nI've added some comments after the player's arrival at Shimmering Lake to\nprompt Code Suggestions to help us build this out:\n\n\n```cpp\n    // If 'choice' is 3, this block is executed.\n    case 3:\n        std::cout \u003C\u003C \"You chose Shimmering Lake.\" \u003C\u003C std::endl;\n        // The player arrives at Shimmering Lake. It is one of the most beautiful lakes the player has ever seen.\n        // The player hears a mysterious melody from the water.\n        // They can either 1. Stay quiet and listen, or 2. Sing along with the melody.\n\n        break;\n```\n\n\nNow, if you start writing `std::cout` to begin offering the player this new\ndecision point, Code Suggestions will help fill out the output code.\n\n\n![adventure.cpp - Code Suggestions helps fill out the output code based on\nthe comments about the interaction at the\nLake](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.4-case-3-output.png){:\n.shadow}\n\n\nYou might find that the code provided by Code Suggestions is very\ndeclarative. Once I've accepted the suggestion, I personalize the code as\nneeded. For example in this case, including the melody the player heard and\nusing the player's name instead of \"you\":\n\n\n![adventure.cpp - I added the playerName to the output and then prompted\nCode Suggestions to continue the narrative based on the comments\nabove](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.4.1-customizing-output.png){:\n.shadow}\n\n\nI also wanted Code Suggestions to offer suggestions in a specific format, so\nI added an end line:\n\n\n![adventure.cpp - I added an end line to prompt Code Suggestions to break\nthe choices into end line\noutputs](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.4.2-customizing-output-endline.png){:\n.shadow}\n\n\n![adventure.cpp - I added an endline to prompt Code Suggestions to break the\nchoices into end line\noutputs](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.4.3-sub-choices-output.png){:\n.shadow}\n\n\nNow, we'd like to offer our player a nested choice in this scenario. Before\nwe can define the new choices, we need a variable to store this nested\nchoice. Let's define a new variable `int nestedChoice` in our `main()`\nfunction, outside of the `switch()` statement we set up. You can put it\nafter our definition of the `choice` variable.\n\n\n```cpp\n    // Declare an int variable to capture the user's choice\n    int choice;\n    // Declare an int variable to capture the user's nested choice\n    int nestedChoice;\n```\n\n\nNext, returning to the `if/else` statement we were working on in `case 3`,\nwe want to prompt the player for their decision and save it in\n`nestedChoice`.\n\n\n![adventure.cpp - I added an end line to prompt Code Suggestions to break\nthe choices into end line\noutputs](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.4.4-capture-nested-choice.png){:\n.shadow}\n\n\nAs you can see, Code Suggestions wants to go ahead and handle the user's\nchoice using another `switch` statement. I would prefer to use an `if/else`\nstatement to handle this decision point.\n\n\nFirst, let's add some comments to give context:\n\n\n```cpp\n    // Capture the user's nested choice\n    std::cin >> nestedChoice;\n\n    // If the player chooses 1 and remains silent, they hear whispers of the merfolk below, but nothing happens.\n    // If the player chooses 2 and sings along, a merfolk surfaces and gifts them a special blue gem as a token of appreciation for their voice.\n\n    // Evaluate the user's nestedChoice\n```\n\n\nThen, start typing `if (nestedChoice == 1)` and Code Suggestions will start\nto offer suggestions:\n\n\n![adventure.cpp - Code Suggestions starts to build out an if statement to\nhandle the\nnestedChoice](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.5-nested-choice-if.png){:\n.shadow}\n\n\nIf you tab to accept them, Code Suggestions will continue to fill out the\nrest of the nested `if/else` statements.\n\n\nSometimes, while you're customizing the suggestions that Code Suggestions\ngives, you may even discover that it would like to make creative\nsuggestions, too!\n\n\n![adventure.cpp - Code Suggestions makes a creative suggestion to end the\ninteraction with the merfolk by saying \"You are now free to go\" after you\nreceive the\ngem.](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.5.2-nested-cs-creative-suggestion.png){:\n.shadow}\n\n\nHere's the code for `case 3` for the player's interaction at Shimmering Lake\nwith the nested decision. I've updated some of the narrative dialogue\nplayer's name.\n\n```\n    // Handle the Shimmering Lake scenario.\n    case 3:\n        std::cout \u003C\u003C playerName \u003C\u003C \" arrives at Shimmering Lake. It is one of the most beautiful lakes that\" \u003C\u003C playerName \u003C\u003C \" has seen. They hear a mysterious melody from the water. They can either: \" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"1. Stay quiet and listen\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"2. Sing along with the melody\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Please enter your choice: \";\n\n        // Capture the user's nested choice\n        std::cin >> nestedChoice;\n\n        // If the player chooses to remain silent\n        if (nestedChoice == 1)\n        {\n            std::cout \u003C\u003C \"Remaining silent, \" \u003C\u003C playerName \u003C\u003C \" hears whispers of the merfolk below, but nothing happens.\" \u003C\u003C std::endl;\n        }\n        // If the player chooses to sing along with the melody\n        else if (nestedChoice == 2)\n        {\n            std::cout \u003C\u003C \"Singing along, a merfolk surfaces and gifts \" \u003C\u003C playerName\n                    \u003C\u003C \" a special blue gem as a token of appreciation for their voice.\"\n                    \u003C\u003C std::endl;\n        }\n        break;\n```\n\n\nOur player isn't limited to just exploring Shimmering Lake. There's a whole\nrealm to explore and they might want to go back and explore other locations.\n\n\nTo facilitate this, we can use a `while` loop. A loop is a type of\nconditional that allows a specific section of code to be executed multiple\ntimes based on a condition. For the `condition` that allows our `while` loop\nto run multiple times, let's use a `boolean` to initialize the loop\ncondition.\n\n\n```cpp\n    // Initialize a flag to control the loop and signify the player's intent to explore.\n    bool exploring = true;\n    // As long as the player wishes to keep exploring, this loop will run.\n    while(exploring) {\n        // wrap the code for switch(choice)\n    }\n```\n\n\nWe also need to move our location prompt inside the `while` loop so that the\nplayer can visit more than one location at the time.\n\n\n![adventure.cpp - CS helps us write a go next prompt for the\nlocations](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.6-while-loop-go-next.png){:\n.shadow}\n\n\n```cpp\n    // Initialize a flag to control the loop and signify the player's intent to explore.\n    bool exploring = true;\n    // As long as the player wishes to keep exploring, this loop will run.\n    while(exploring) {\n\n        // If still exploring, ask the player where they want to go next\n        std::cout \u003C\u003C \"Where will \" \u003C\u003C playerName \u003C\u003C \" go next?\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"1. Moonlight Markets\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"2. Grand Library\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"3. Shimmering Lake\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Please enter your choice: \";\n        // Update value of choice\n        std::cin >> choice;\n\n        // Respond based on the player's main choice\n        switch(choice) {\n```\n\n\nOur `while` loop will keep running as long as `exploring` is `true`, so we\nneed a way for the player to have the option to exit the game. Let's add a\ncase 4 that allows the player to exit by setting `exploring = false`. This\nwill exit the loop and take the player back to the original choices.\n\n\n```cpp\n    // Option to exit the game\n    case 4:\n        exploring = false;\n        break;\n```\n\n\n**Async exercise**: Give the player the option to exit the game instead of\nexploring a new decision.\n\n\nWe also need to update the error handling for invalid inputs in the `switch`\nstatement. You can decide whether to end the program or use the `continue`\nstatement to start a new loop iteration.\n\n\n```cpp\n        default:\n            std::cout \u003C\u003C \"You did not enter a valid choice.\" \u003C\u003C std::endl;\n            continue; // Errors continue with the next loop iteration\n```\n\n\nUsing I/O and conditionals is at the core of text-based adventure games and\nhelps make these games interactive. We can combine user input, display\noutput, and implement our narrative into decision-making logic to create an\nengaging experience.\n\n\nHere's what our `adventure.cpp` looks like now with some comments:\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    std::cout \u003C\u003C \"Welcome to the Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Declare a string variable to store the player name\n    std::string playerName;\n\n    // Prompt the user to enter their player name\n    std::cout \u003C\u003C \"Please enter your name: \";\n    std::cin >> playerName;\n\n    // Display a personalized welcome message to the player with their name\n    std::cout \u003C\u003C \"Welcome \" \u003C\u003C playerName \u003C\u003C \" to The Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Declare an int variable to capture the user's choice\n    int choice;\n    // Declare an int variable to capture the user's nested choice\n    int nestedChoice;\n\n    // Initialize a flag to control the loop and signify the player's intent to explore.\n    bool exploring = true;\n    // As long as the player wishes to keep exploring, this loop will run.\n    while(exploring) {\n\n        // If still exploring, ask the player where they want to go next\n        std::cout \u003C\u003C \"Where will \" \u003C\u003C playerName \u003C\u003C \" go next?\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"1. Moonlight Markets\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"2. Grand Library\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"3. Shimmering Lake\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Please enter your choice: \";\n        // Update value of choice\n        std::cin >> choice;\n\n        // Respond based on the player's main choice\n        switch(choice) {\n            //  Handle the Moonlight Markets scenario\n            case 1:\n                std::cout \u003C\u003C \"You chose Moonlight Markets.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Grand Library scenario.\n            case 2:\n                std::cout \u003C\u003C \"You chose Grand Library.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Shimmering Lake scenario.\n            case 3:\n                std::cout \u003C\u003C playerName \u003C\u003C \" arrives at Shimmering Lake. It is one of the most beautiful lakes that\" \u003C\u003C playerName \u003C\u003C \" has seen. They hear a mysterious melody from the water. They can either: \" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"1. Stay quiet and listen\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"2. Sing along with the melody\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"Please enter your choice: \";\n\n                // Capture the user's nested choice\n                std::cin >> nestedChoice;\n\n                // If the player chooses to remain silent\n                if (nestedChoice == 1)\n                {\n                    std::cout \u003C\u003C \"Remaining silent, \" \u003C\u003C playerName \u003C\u003C \" hears whispers of the merfolk below, but nothing happens.\" \u003C\u003C std::endl;\n                }\n                // If the player chooses to sing along with the melody\n                else if (nestedChoice == 2)\n                {\n                    std::cout \u003C\u003C \"Singing along, a merfolk surfaces and gifts \" \u003C\u003C playerName\n                            \u003C\u003C \" a special blue gem as a token of appreciation for their voice.\"\n                            \u003C\u003C std::endl;\n                }\n                break;\n            // Option to exit the game\n            case 4:\n                exploring = false;\n                break;\n            // If 'choice' is not 1, 2, or 3, this block is executed.\n            default:\n                std::cout \u003C\u003C \"You did not enter a valid choice.\" \u003C\u003C std::endl;\n                continue; // Errors continue with the next loop iteration\n        }\n    }\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\nHere's what the build output looks like if we run `adventure.cpp` and the\nplayer heads to the Shimmering Lake.\n\n\n![adventure.cpp build output - the player is called sugaroverflow and heads\nto the Shimmering Lake and receives a\ngem](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/3.6.1-full-case-3-output.png){:\n.shadow}\n\n\n## Structuring the narrative: Characters\n\nOur player can now explore the world. Soon, our player will also be able to\nmeet people and collect objects. Before we can do that, let's organize the\nthings our player can do with creating some structure for the player\ncharacter.\n\n\nIn C++, a `struct` is used to group different data types. It's helpful in\ncreating a group of items that belong together, such as our player's\nattributes and inventory, into a single unit. `struct` objects are defined\nglobally, which means at top the file, before the `main() function.\n\n\nIf you start typing `struct Player {`, Code Suggestions will help you out\nwith a sample definition of a player struct.\n\n\n![adventure.cpp - Code Suggestions helps with setting up the struct\ndefinition for the\nplayer](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/4-player-struct-definition.png){:\n.shadow}\n\n\nAfter accepting this suggestion, you might find that Code Suggestions is\neager to define some functions to make this game more fun, such as hunting\nfor treasure.\n\n\n![adventure.cpp - Code Suggestions provides a suggestion for creating\nfunctions to hunt for\ntreasure.](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/4.1-player-struct-treasure-suggestion.png){:\n.shadow}\n\n\n```cpp\n\n// Define a structure for a Player in the game.\n\nstruct Player{\n    std::string name;  // The name of the player.\n    int health;        // The current health of the player.\n    int xp;            // Experience points gained by the player. Could be used for leveling up or other game mechanics.\n};\n\n```\n\n\nGiving the player experience points was not in my original plan for this\ntext adventure game, but Code Suggestions makes an interesting suggestion.\nWe could use `xp` for leveling up or for other game mechanics as our project\ngrows.\n\n\n`struct Player` provides a blueprint for creating a player and details the\nattributes that make up a player. To use our player in our code, we must\ninstantiate, or create, an object of the `Player` struct within our `main()`\nfunction. Objects in C++ are instances of structures that contain\nattributes. In our example, we're working with the `Player` struct, which\nhas attributes like name, health, and xp.\n\n\nAs you're creating a `Player` object, you might find that Code Suggestions\nwants to name the player \"John.\"\n\n\n![adventure.cpp - code suggestions suggests naming the new Player object\nJohn.](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/4.2-player-struct-instance-john.png){:\n.shadow}\n\n\n```cpp\n\nint main() {\n    // Create an instance of the Player struct\n    Player player;\n    player.health = 100; // Assign a default value for HP\n```\n\n\nInstead of naming our player \"John\" for everyone, we'll use the `Player`\nobject to set the attribute for name. When we want to interact with or\nmanipulate an attribute of an object, we use the dot operator `.`. The dot\noperator allows us to access specific members of the object. We can set the\nplayer's name using the dot operator with `player.name`.\n\n\nNote that we need to replace other mentions of `playerName` the variable\nwith `player.name`, which allows us to access the player object's name\ndirectly.\n\n\n* Search for all occurrences of the `playerName` variable, and replace it\nwith `player.name`.\n\n* Comment/Remove the unused `std::string playerName` variable after that.\n\n\nWhat your `adventure.cpp` will look like now:\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Define a structure for a Player in the game.\n\nstruct Player{\n    std::string name;  // The name of the player.\n    int health;        // The current health of the player.\n    int xp;            // Experience points gained by the player. Could be used for leveling up or other game mechanics.\n};\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    std::cout \u003C\u003C \"Welcome to the Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Create an instance of the Player struct\n    Player player;\n    player.health = 100; // Assign a default value for HP\n\n    // Prompt the user to enter their player name\n    std::cout \u003C\u003C \"Please enter your name: \";\n    std::cin >> player.name;\n\n    // Display a personalized welcome message to the player with their name\n    std::cout \u003C\u003C \"Welcome \" \u003C\u003C player.name \u003C\u003C \" to The Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Declare an int variable to capture the user's choice\n    int choice;\n    // Declare an int variable to capture the user's nested choice\n    int nestedChoice;\n\n    // Initialize a flag to control the loop and signify the player's intent to explore.\n    bool exploring = true;\n    // As long as the player wishes to keep exploring, this loop will run.\n    while(exploring) {\n\n        // If still exploring, ask the player where they want to go next\n        std::cout \u003C\u003C \"Where will \" \u003C\u003C player.name \u003C\u003C \" go next?\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"1. Moonlight Markets\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"2. Grand Library\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"3. Shimmering Lake\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Please enter your choice: \";\n        // Update value of choice\n        std::cin >> choice;\n\n        // Respond based on the player's main choice\n        switch(choice) {\n            //  Handle the Moonlight Markets scenario\n            case 1:\n                std::cout \u003C\u003C \"You chose Moonlight Markets.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Grand Library scenario.\n            case 2:\n                std::cout \u003C\u003C \"You chose Grand Library.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Shimmering Lake scenario.\n            case 3:\n                std::cout \u003C\u003C player.name \u003C\u003C \" arrives at Shimmering Lake. It is one of the most beautiful lakes that\" \u003C\u003C player.name \u003C\u003C \" has seen. They hear a mysterious melody from the water. They can either: \" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"1. Stay quiet and listen\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"2. Sing along with the melody\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"Please enter your choice: \";\n\n                // Capture the user's nested choice\n                std::cin >> nestedChoice;\n\n                // If the player chooses to remain silent\n                if (nestedChoice == 1)\n                {\n                    std::cout \u003C\u003C \"Remaining silent, \" \u003C\u003C player.name \u003C\u003C \" hears whispers of the merfolk below, but nothing happens.\" \u003C\u003C std::endl;\n                }\n                // If the player chooses to sing along with the melody\n                else if (nestedChoice == 2)\n                {\n                    std::cout \u003C\u003C \"Singing along, a merfolk surfaces and gifts \" \u003C\u003C player.name\n                            \u003C\u003C \" a special blue gem as a token of appreciation for their voice.\"\n                            \u003C\u003C std::endl;\n                }\n                break;\n            // Option to exit the game\n            case 4:\n                exploring = false;\n                break;\n            // If 'choice' is not 1, 2, or 3, this block is executed.\n            default:\n                std::cout \u003C\u003C \"You did not enter a valid choice.\" \u003C\u003C std::endl;\n                continue; // Errors continue with the next loop iteration\n        }\n    }\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\n## Structuring the narrative: Items\n\nAn essential part of adventure games is a player's inventory - the\ncollection of items they acquire and use during their journey. For example,\nat Shimmering Lake, the player acquired a blue gem.\n\n\nLet's update our Player `struct` to include an inventory using an array. In\nC++, an `array` is a collection of elements of the same type that can be\nidentified by an index. When creating an array, you need to specify its type\nand size. Start by adding `std::string inventory` to the Player `struct`:\n\n\n![adventure.cpp - Code Suggestions shows us how to add an array of strings\nto the player struct to use as the players\ninventory](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5-add-inventory-player-struct.png){:\n.shadow}\n\n\nYou might find that Code Suggestions wants our player to be able to carry\nsome gold, but we don't need that for now. Let's also add `int\ninventoryCount;` to keep track of the number of items in our player's\ninventory.\n\n\n![adventure.cpp - Code Suggestions shows us how to add an integer for\ninventoryCount to the player\nstruct](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5.1-add-inventory-count-player-struct.png){:\n.shadow}\n\n\n```cpp\n\n// Define a structure for a Player in the game.\n\nstruct Player{\n    std::string name;  // The name of the player.\n    int health;        // The current health of the player.\n    int xp;            // Experience points gained by the player. Could be used for leveling up or other game mechanics.\n    std::string inventory[10];  // An array of strings for the player's inventory.\n    int inventoryCount = 0;  // The number of items in the player's inventory.\n};\n\n```\n\nIn our Player `struct`, we have defined an array for our inventory that can\nhold the names of 10 items (type:string, size: 10). As the player progresses\nthrough our story, we can assign new items to the inventory array based on\nthe player's actions using the array index.\n\n\nSometimes Code Suggestions gets ahead of me and tries to add more complexity\nto the game by suggesting that we need to create a `struct` for some\nMonsters. Maybe later, Code Suggestions!\n\n\n![adventure.cpp - Code Suggestions wants to add a struct for Monsters we can\nbattle](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5.2-suggestion-gets-distracted-by-monsters.png\n\n){: .shadow}\n\n\nBack at the Shimmering Lake, the player received a special blue gem from the\nmerfolk. Let's update the code in `case 2` for the Shimmering Lake to add\nthe gem to our player's inventory.\n\n\nYou can start by accessing the player's inventory with `player.inventory`\nand Code Suggestions will help add the gem.\n\n\n![adventure.cpp - Code Suggestions shows us how to add a gem to the player's\ninventory using a post-increment operation and the inventory array from the\nstruct\nobject](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5.3-add-gem-to-inventory.png){:\n.shadow}\n\n\n```cpp\n    // If the player chooses to sing along with the melody\n    else if (nestedChoice == 2)\n    {\n        std::cout \u003C\u003C \"Singing along, a merfolk surfaces and gifts \" \u003C\u003C player.name\n                \u003C\u003C \" a special blue gem as a token of appreciation for their voice.\"\n                \u003C\u003C std::endl;\n        player.inventory[player.inventoryCount] = \"Blue Gem\";\n        player.inventoryCount++;\n    }\n```\n\n\n* `player.inventory`: accesses the inventory attribute of the player object\n\n* `player.inventoryCount`: accesses the integer that keeps track of how many\nitems are currently in the player's inventory. This also represents the next\navailable index in our inventory array where an item can be stored.\n\n* `player.inventoryCount++`: increments the value of inventoryCount by 1.\nThis is a post-increment operation. We are adding “Blue Gem” to the next\navailable slot in the inventory array and incrementing the array for the\nnewly added item.\n\n\nOnce we've added something to our player's inventory, we may also want to be\nable to look at everything in the inventory. We can use a `for` loop to\niterate over the inventory array and display each item.\n\n\nIn C++, a `for` loop allows code to be repeatedly executed a specific number\nof times. It's different from the `while` loop we used earlier because the\n`while` executes its body based on a condition, whereas a `for` loop\niterates over a sequence or range, usually with a known number of times.\n\n\nAfter adding the gem to the player's inventory, let's display all the items\nit has. Try starting a for loop with `for ( ` to display the player's\ninventory and Code Suggestions will help you with the syntax.\n\n\n![adventure.cpp - Code Suggestions demonstrates how to write a for loop to\nloop through the players\ninventory](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5.4-loop-over-players-inventory.png){:\n.shadow}\n\n\n```cpp\n\nstd::cout \u003C\u003C player.name \u003C\u003C \"'s Inventory:\" \u003C\u003C std::endl;\n\n// Loop through the player's inventory up to the count of items they have\n\nfor (int i = 0; i \u003C player.inventoryCount; i++)\n\n{\n    // Output the item in the inventory slot\n    std::cout \u003C\u003C \"- \" \u003C\u003C player.inventory[i] \u003C\u003C std::endl;\n}\n\n```\n\n\nA `for` loop consists of 3 main parts:\n\n\n* `int i = 0`: is the initialization where you set up your loop variable.\nHere, we start counting from 0.\n\n* `i \u003C player.inventoryCount`: is the condition we're looping on, our loop\nchecks if `i`, the current loop variable, is less than the number of items\nin our inventory. It will keep going until this is true.\n\n* `i++`: is the iteration. This updates the loop variable each time the loop\nruns.\n\n\nTo make sure that our loop doesn't encounter an error, let's add some error\nhandling to make sure the inventory is not empty when we try to output it.\n\n\n```\n\nstd::cout \u003C\u003C player.name \u003C\u003C \"'s Inventory:\" \u003C\u003C std::endl;\n\n// Loop through the player's inventory up to the count of items they have\n\nfor (int i = 0; i \u003C player.inventoryCount; i++)\n\n{\n    // Check if the inventory slot is not empty.\n    if (!player.inventory[i].empty())\n    {\n        // Output the item in the inventory slot\n        std::cout \u003C\u003C \"- \" \u003C\u003C player.inventory[i] \u003C\u003C std::endl;\n    }\n}\n\n```\n\n\nWith our progress so far, we've successfully established a persistent\n`while` loop for our adventure, handled decisions, crafted a `struct` for\nour player, and implemented a simple inventory system. Now, let's dive into\nthe next scenario, the Grand Library, applying the foundations we've\nlearned.\n\n\n**Async exercise**: Add more inventory items found in different locations.\n\n\nHere's what we have for `adventure.cpp` so far:\n\n\n```cpp\n\n#include \u003Ciostream> // Include the I/O stream library for input and output\n\n\n// Define a structure for a Player in the game.\n\nstruct Player{\n    std::string name;  // The name of the player.\n    int health;        // The current health of the player.\n    int xp;            // Experience points gained by the player. Could be used for leveling up or other game mechanics.\n    std::string inventory[10];  // An array of strings for the player's inventory.\n    int inventoryCount = 0;\n};\n\n\n// Main function, the starting point of the program\n\nint main()\n\n{\n    std::cout \u003C\u003C \"Welcome to the Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Create an instance of the Player struct\n    Player player;\n    player.health = 100; // Assign a default value for HP\n\n    // Prompt the user to enter their player name\n    std::cout \u003C\u003C \"Please enter your name: \";\n    std::cin >> player.name;\n\n    // Display a personalized welcome message to the player with their name\n    std::cout \u003C\u003C \"Welcome \" \u003C\u003C player.name \u003C\u003C \" to The Dragon Realm!\" \u003C\u003C std::endl;\n\n    // Declare an int variable to capture the user's choice\n    int choice;\n    // Declare an int variable to capture the user's nested choice\n    int nestedChoice;\n\n    // Initialize a flag to control the loop and signify the player's intent to explore.\n    bool exploring = true;\n    // As long as the player wishes to keep exploring, this loop will run.\n    while(exploring) {\n\n        // If still exploring, ask the player where they want to go next\n        std::cout \u003C\u003C \"--------------------------------------------------------\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Where will \" \u003C\u003C player.name \u003C\u003C \" go next?\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"1. Moonlight Markets\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"2. Grand Library\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"3. Shimmering Lake\" \u003C\u003C std::endl;\n        std::cout \u003C\u003C \"Please enter your choice: \";\n        // Update value of choice\n        std::cin >> choice;\n\n        // Respond based on the player's main choice\n        switch(choice) {\n            //  Handle the Moonlight Markets scenario\n            case 1:\n                std::cout \u003C\u003C \"You chose Moonlight Markets.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Grand Library scenario.\n            case 2:\n                std::cout \u003C\u003C \"You chose Grand Library.\" \u003C\u003C std::endl;\n                break;\n            // Handle the Shimmering Lake scenario.\n            case 3:\n                std::cout \u003C\u003C player.name \u003C\u003C \" arrives at Shimmering Lake. It is one of the most beautiful lakes that\" \u003C\u003C player.name \u003C\u003C \" has seen. They hear a mysterious melody from the water. They can either: \" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"1. Stay quiet and listen\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"2. Sing along with the melody\" \u003C\u003C std::endl;\n                std::cout \u003C\u003C \"Please enter your choice: \";\n\n                // Capture the user's nested choice\n                std::cin >> nestedChoice;\n\n                // If the player chooses to remain silent\n                if (nestedChoice == 1)\n                {\n                    std::cout \u003C\u003C \"Remaining silent, \" \u003C\u003C player.name \u003C\u003C \" hears whispers of the merfolk below, but nothing happens.\" \u003C\u003C std::endl;\n                }\n                // If the player chooses to sing along with the melody\n                else if (nestedChoice == 2)\n                {\n                    std::cout \u003C\u003C \"Singing along, a merfolk surfaces and gifts \" \u003C\u003C player.name\n                            \u003C\u003C \" a special blue gem as a token of appreciation for their voice.\"\n                            \u003C\u003C std::endl;\n                    player.inventory[player.inventoryCount] = \"Blue Gem\";\n                    player.inventoryCount++;\n\n                    std::cout \u003C\u003C player.name \u003C\u003C \"'s Inventory:\" \u003C\u003C std::endl;\n                    // Loop through the player's inventory up to the count of items they have\n                    for (int i = 0; i \u003C player.inventoryCount; i++)\n                    {\n                        // Check if the inventory slot is not empty.\n                        if (!player.inventory[i].empty())\n                        {\n                            // Output the item in the inventory slot\n                            std::cout \u003C\u003C \"- \" \u003C\u003C player.inventory[i] \u003C\u003C std::endl;\n                        }\n                    }\n\n                }\n                break;\n            // Option to exit the game\n            case 4:\n                exploring = false;\n                break;\n            // If 'choice' is not 1, 2, or 3, this block is executed.\n            default:\n                std::cout \u003C\u003C \"You did not enter a valid choice.\" \u003C\u003C std::endl;\n                continue; // Errors continue with the next loop iteration\n        }\n    }\n\n    // Return 0 to indicate successful execution\n    return 0;\n}\n\n```\n\n\n![adventure.cpp - A full output of the game at the current state - our\nplayer sugaroverflow visits the Lake, receives the gem, adds it to their\ninventory, and we display the inventory before returning to the\nloop](https://about.gitlab.com/images/blogimages/2023-08-21-building-a-text-adventure-using-cplusplus-and-code-suggestions/5.5-full-output-shimmering-lake.png){:\n.shadow}\n",[495,720,956,851,9],{"slug":1222,"featured":6,"template":700},"building-a-text-adventure-using-cplusplus-and-code-suggestions","content:en-us:blog:building-a-text-adventure-using-cplusplus-and-code-suggestions.yml","Building A Text Adventure Using Cplusplus And Code Suggestions","en-us/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions.yml","en-us/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions",{"_path":1228,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1229,"content":1235,"config":1242,"_id":1244,"_type":13,"title":1245,"_source":15,"_file":1246,"_stem":1247,"_extension":18},"/en-us/blog/building-build-images",{"title":1230,"description":1231,"ogTitle":1230,"ogDescription":1231,"noIndex":6,"ogImage":1232,"ogUrl":1233,"ogSiteName":685,"ogType":686,"canonicalUrls":1233,"schema":1234},"Getting [meta] with GitLab CI/CD: Building build images","Let's talk about building build images with GitLab CI/CD. The power of Docker as a build platform is unleashed when you get meta.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749678567/Blog/Hero%20Images/building-blocks.jpg","https://about.gitlab.com/blog/building-build-images","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting [meta] with GitLab CI/CD: Building build images\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Brendan O'Leary\"}],\n        \"datePublished\": \"2019-08-28\",\n      }",{"title":1230,"description":1231,"authors":1236,"heroImage":1232,"date":1238,"body":1239,"category":785,"tags":1240},[1237],"Brendan O'Leary","2019-08-28","> An alternative title for this post could have been:\n\n>\n\n> I heard you liked Docker, so I put\n[dind](https://hub.docker.com/_/docker/).\n\n\n## Getting started\n\nIt should be clear by now that I love building stuff with GitLab CI/CD. From\n\n[DNS](https://medium.com/gitlab-magazine/ci-cd-all-the-things-pihole-625a0ceaf12)\n\nto [breakfast](/blog/introducing-auto-breakfast-from-gitlab/) GitLab CI/CD\n\noffers a pretty wide range. However, past those \"fun\" use cases, I also like\n\nto share some ~~best~~ practices I have acquired during my years of using\n[GitLab\n\nCI/CD](/solutions/continuous-integration/), both for software and\nnon-software projects alike.\n\n\nI crossed out \"best\" above because I don't really like the term \"best\npractices.\" It\n\nimplies that there is only one right answer to a given question – which is\nthe\n\nopposite of the point of computer science. Sure there are better and worse\nways to\n\ndo something – but like many things in life, you have to find what works for\n\nyou. \"[The best camera is the one you have with\nyou](https://www.amazon.com/Best-Camera-One-Thats-You/dp/0321684788)\"\n\ncomes to mind when building CI/CD for projects. Something that works is\nbetter than something that's pretty.\n\n\nBut, enough of my digression, let's get to the practice I wanted to share in\nthis\n\npost: Building build images as part of the build process. Yes, it is\nprecisely as meta as it sounds.\n\n\n## Why?\n\n\nOften when building a particular project, you may have several unique build\ndependencies.\n\nIn many languages, package managers solve for the majority if not all of\nthese\n\ndependencies – at least for build time (think [npm](https://www.npmjs.com),\n[RubyGems](https://rubygems.org/),\n\n[Maven](https://maven.apache.org/what-is-maven.html)). However, when we are\nbuilding and\n\ndeploying (CI/**CD** let's remember) from a machine that is not our own,\nthat may not\n\nbe enough. There may be a few dependencies we might need from elsewhere.\n\n\nThe language libraries themselves are one such dependency – to build Java\nI'm going to need\n\nthe JDK or JRE. To build Node, I'll need... well Node, etc. In a\nDocker-based environment,\n\nthose languages and dependencies typically have an official image on Docker\n\nHub ([JRE from Oracle](https://hub.docker.com/_/oracle-serverjre-8) or\n\n[Node from Node.js](https://hub.docker.com/_/node) for instance). Assume,\nhowever, that\n\nI may need a few other things not included in **either** those official\nDocker images or\n\nthe package manager I'm using. For instance, maybe I need a CLI tool for\n\ndeploy ([AWS](https://aws.amazon.com/cli/),\n[Heroku](https://devcenter.heroku.com/articles/heroku-cli),\n\n[Firebase](https://firebase.google.com/docs/cli), etc.). We also might need\na testing\n\nframework or tool like [Selenium](https://www.seleniumhq.org) or\n\n[headless\nChrome](https://developers.google.com/web/updates/2017/04/headless-chrome).\n\nOr I may need other tools for packaging, testing, or deployment.\n\n\nSometimes there is a Docker image on Docker Hub for these combinations – or\nsome of\n\nthem – but not always a maintained version. One easy solution to this could\nbe to\n\njust run the install of the tools before every job that needs it. This can\n\neven be \"automated\" using something like\n\nthe\n[before_script](https://docs.gitlab.com/ee/ci/yaml/#before_script-and-after_script)\nsyntax.\n\nHowever, this adds time to our pipeline and seems inefficient: Is there a\nbetter way?\n\n\n## Enter the GitLab Docker registry\n\nSince GitLab is a single application for the entire\n[DevOps](/topics/devops/) lifecycle – it actually\n\nships out of the box with a built-in\n\n[Docker\nregistry](https://docs.gitlab.com/ee/user/packages/container_registry/index.html).\n\nThis can be a useful tool when deploying code in a containerized\nenvironment. We can\n\nbuild our application into a container and send it off into Kubernetes or\nsome\n\nother Docker orchestrator.\n\n\nHowever, I also see this registry as an opportunity to save time in my\n\npipeline (and save round trips to Docker hub and back every time). For\nbuilds that require\n\nsome of these extra dependencies, I like to build a \"build\" Docker image.\n\nThat way, I have an image with all of those baked right in. Then, as part of\nmy\n\npipeline, I can build the image at the start (only when changes are made or\nevery time).\n\nAnd the rest of the pipeline can consume that image as the base image.\n\n\n## Putting it in practice\n\nFor example, let's see what it looks like to build a simple Docker image to\nuse with\n\ndeploying to [Google Firebase](https://firebase.google.com/).\n\n\nFirebase is a \"backend as a service\" tool that provides a database,\nauthentication,\n\nand other services across platforms (web, iOS, and Android). It also\nincludes web hosting\n\nand several other items that can be deployed through [a\nCLI](https://firebase.google.com/docs/cli).\n\nThis tool makes getting started really easy. You can deploy the whole stack\nwith\n\n`firebase deploy.` Alternatively, you can deploy a part (like\n[serverless](/topics/serverless/) functions)\n\nwith a command like `firebase deploy --only functions.`\n\n\nMaking this work in a CI/CD world requires a few extra steps though. We'll\nneed a Node\n\nDocker image that has the firebase CLI in it, so let's make a simple\nDockerfile to do that.\n\n\n> Putting this Dockerfile in `.meta/Dockerfile`\n\n\n```dockerfile\n\nFROM node:10\n\n\nRUN npm install -g firebase-tools\n\n```\n\n\nNext, I'll add a job to the front of my pipeline.\n\n\n> Added to the front of my `.gitlab-ci.yml`\n\n\n```yaml\n\nmeta-build-image:\n  image: docker:stable\n  services:\n    - docker:dind\n  stage: prepare\n  script:\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - cd .meta\n    - docker build -t $CI_REGISTRY/group/project/buildimage:latest .\n    - docker push $CI_REGISTRY/group/project/buildimage:latest\n  only:\n    refs:\n      - main\n    changes:\n      - .meta/Dockerfile\n```\n\n\nLet's break down that job:\n\n1. We use the `docker:stable` image and a service of `docker:dind`\n\n1. The stage is my first stage called `prepare`\n\n1. In the script, we login to the GitLab registry with the built-in\nvariables and build the\n\nimage. For more details see the [GitLab documentation for building Docker\nimages](https://docs.gitlab.com/ee/ci/docker/using_docker_build.html).\n\n1. We only run this on `main` and only when the `.meta/Dockerfile` changes.\nThis makes\n\nsure we are specific about when we change the Docker image. We could also\nuse the\n\ncommit hash or other methods here to make the image more fungible.\n\n\nNow, in further jobs down the pipeline, I can use the latest build of the\nDocker image like this:\n\n\n```yaml\n\nfirestore:\n  image: registry.gitlab.com/group/project/buildimage\n  stage: deploy 🚢🇮🇹\n  script:\n    - firebase deploy --only firestore\n  only:\n    changes:\n      - .firebase-config/firestore.rules\n      - .firebase-config/firestore.indexes.json\n```\n\n\nIn this job, we only run the job if something about\n\nthe [Firestore](https://firebase.google.com/docs/firestore) (the database\nfrom Firebase)\n\nconfiguration changes. And when it does, we run the `firestore deploy`\ncommand in CI. I\n\nalso added a token for deploy as a [GitLab CI/CD\nvariable](https://docs.gitlab.com/ee/ci/variables/)\n\nbased off the Firebase documentation\n\nfor [using firebase with\nCI](https://firebase.google.com/docs/cli#admin-commands).\n\n\n## Summary\n\nIn the end, this helps speed up pipelines by ensuring that you have a\ncustom-built build\n\nimage that you control. You don't have to rely on unstable or unmaintained\nDocker Hub\n\nimages or even have a Docker Hub account yourself to get started.\n\n\nTo learn more about GitLab CI/CD you can [read the GitLab\nwebsite](/solutions/continuous-integration/)\n\nor the [CI/CD docs](https://docs.gitlab.com/ee/ci/introduction/). Also,\nthere's a lot more to\n\nlearn about the [GitLab Docker\nregistry](https://docs.gitlab.com/ee/user/packages/container_registry/index.html).\n\n\nCover image by [Hack\nCapital](https://unsplash.com/@markusspiske?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/search/photos/build?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText).\n\n{: .note}\n",[108,1241,9],"kubernetes",{"slug":1243,"featured":6,"template":700},"building-build-images","content:en-us:blog:building-build-images.yml","Building Build Images","en-us/blog/building-build-images.yml","en-us/blog/building-build-images",{"_path":1249,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1250,"content":1256,"config":1265,"_id":1267,"_type":13,"title":1268,"_source":15,"_file":1269,"_stem":1270,"_extension":18},"/en-us/blog/building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features",{"title":1251,"description":1252,"ogTitle":1251,"ogDescription":1252,"noIndex":6,"ogImage":1253,"ogUrl":1254,"ogSiteName":685,"ogType":686,"canonicalUrls":1254,"schema":1255},"Building GitLab with GitLab: A multi-region service to deliver AI features","Discover how we built our first multi-region deployment for teams at GitLab using the platform's many features, helping create a frictionless developer experience for GitLab Duo users.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098664/Blog/Hero%20Images/Blog/Hero%20Images/building-gitlab-with-gitlab-no-type_building-gitlab-with-gitlab-no-type.png_1750098663794.png","https://about.gitlab.com/blog/building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Building GitLab with GitLab: A multi-region service to deliver AI features\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Chance Feick\"},{\"@type\":\"Person\",\"name\":\"Sam Wiskow\"}],\n        \"datePublished\": \"2024-09-12\",\n      }",{"title":1251,"description":1252,"authors":1257,"heroImage":1253,"date":1260,"body":1261,"category":785,"tags":1262},[1258,1259],"Chance Feick","Sam Wiskow","2024-09-12","For GitLab Duo, real-time AI-powered capabilities like [Code\nSuggestions](https://about.gitlab.com/solutions/code-suggestions/) need\nlow-latency response times for a frictionless developer experience. Users\ndon’t want to interrupt their flow and wait for a code suggestion to show\nup. To ensure GitLab Duo can provide the right suggestion at the right time\nand meet high performance standards for critical AI infrastructure, GitLab\nrecently launched our first multi-region service to deliver AI features.\n\n\nIn this article, we will cover the benefits of multi-region services, how we\nbuilt an internal platform codenamed ‘Runway’ for provisioning and deploying\nmulti-region services using GitLab features, and the lessons learned\nmigrating to multi-region in production.\n\n\n## Background on the project\n\n\nRunway is GitLab’s internal platform as a service (PaaS) for provisioning,\ndeploying, and operating containerized services. Runway's purpose is to\nenable GitLab service owners to self-serve infrastructure needs with\nproduction readiness out of the box, so application developers can focus on\nproviding value to customers. As part of [our corporate value of\ndogfooding](https://handbook.gitlab.com/handbook/values/#results), the first\niteration was built in 2023 by the Infrastructure department on top of core\nGitLab capabilities, such as continuous integration/continuous delivery\n([CI/CD](https://about.gitlab.com/topics/ci-cd/)), environments, and\ndeployments.\n\n\nBy establishing automated GitOps best practices, Runway services use\ninfrastructure as code (IaC), merge requests (MRs), and CI/CD by default.\n\n\nGitLab Duo is primarily powered by [AI\nGateway](https://gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist),\na satellite service written in Python outside of GitLab’s modular monolith\nwritten in Ruby. In cloud computing, a region is a geographical location of\ndata centers operated by cloud providers.\n\n\n## Defining a multi-region strategy\n\n\nDeploying in a single region is a good starting point for most services, but\ncan come with downsides when you are trying to reach a global audience.\nUsers who are geographically far from where your service is deployed may\nexperience different levels of service and responsiveness than those who are\ncloser. This can lead to a poor user experience, even if your service is\nwell built in all other respects.\n\n\nFor AI Gateway, it was important to meet global customers wherever they are\nlocated, whether on GitLab.com or self-managed instances using Cloud\nConnector. When a developer is deciding to accept or reject a code\nsuggestion, milliseconds matter and can define the user experience.\n\n\n### Goals\n\n\nMulti-region deployments require more infrastructure complexity, but for use\ncases where latency is a core component of the user experience, the benefits\noften outweigh the downsides. First, multi-region deployments offer\nincreased responsiveness to the user. By serving requests from locations\nclosest to end users, latency can be significantly reduced. Second,\nmulti-region deployments provide greater availability. With fault tolerance,\nservices can fail over during a regional outage. There is a much lower\nchance of a service failing completely, meaning users should not be\ninterrupted even in partial failures.\n\n\nBased on our goals for performance and availability, we used this\nopportunity to create a scalable multi-region strategy in Runway, which is\nbuilt leveraging GitLab features.\n\n\n### Architecture\n\n\nIn SaaS platforms, GitLab.com’s infrastructure is hosted on Google Cloud\nPlatform (GCP). As a result, Runway’s first supported platform runtime is\nCloud Run. The initial workloads deployed on Runway are stateless satellite\nservices (e.g., AI Gateway), so Cloud Run services are a good fit that\nprovide a clear migration path to more complex and flexible platform\nruntimes, e.g. Kubernetes.\n\n\nBuilding Runway on top of GCP Cloud Run using GitLab has allowed us to\niterate and tease out the right level of abstractions for service owners as\npart of a platform play in the Infrastructure department.\n\n\nTo serve traffic from multiple regions in Cloud Run, the multi-region\ndeployment strategy must support global load balancing, and the provisioning\nand configuration of regional resources. Here’s a simplified diagram of the\nproposed architecture in GCP:\n\n\n![simplified diagram of the proposed architecture in\nGCP](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098671/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098671612.png)\n\n\nBy replicating Cloud Run services across multiple regions and configuring\nthe existing global load balancing with serverless network endpoint group\n(NEG) backends, we’re able to serve traffic from multiple regions. For the\nremainder of the article, we’ll focus less on specifics of Cloud Run and\nmore on how we’re building with GitLab.\n\n\n## Building a multi-region platform with GitLab\n\n\nNow that you have context about Runway, let's walk through how to build a\nmulti-region platform using GitLab features.\n\n\n### Provision\n\n\nWhen building an internal platform, the first challenge is provisioning\ninfrastructure for a service. In Runway, Provisioner is the component that\nis responsible for maintaining a service inventory and managing IaC for GCP\nresources using Terraform.\n\n\nTo provision a service, an application developer will open an MR to add a\nservice project to the inventory using git, and Provisioner will create\nrequired resources, such as service accounts and identity and access\nmanagement policies. When building this functionality with GitLab, Runway\nleverages [OpenID Connect (OIDC) with GPC Workload Identity\nFederation](https://docs.gitlab.com/ee/ci/cloud\\_services/google\\_cloud/)\nfor managing IaC.\n\n\nAdditionally, Provisioner will create a deployment project for each service\nproject. The purpose of creating separate projects for deployments is to\nensure the [principle of least\nprivilege](https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab/)\nby authenticating as a GCP service account with restricted permissions.\nRunway leverages the [Projects\nAPI](https://docs.gitlab.com/ee/api/projects.html) for creating projects\nwith [Terraform\nprovider](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs).\n\n\nFinally, Provisioner defines variables in the deployment project for the\nservice account, so that deployment CI jobs can authenticate to GCP. Runway\nleverages [CI/CD variables](https://docs.gitlab.com/ee/ci/variables/) and\n[Job Token\nallowlist](https://docs.gitlab.com/ee/ci/jobs/ci\\_job\\_token.html\\#add-a-group-or-project-to-the-job-token-allowlist)\nto handle authentication and authorization.\n\n\nHere’s a simplified example of provisioning a multi-region service in the\nservice inventory:\n\n\n```\n\n{\n  \"inventory\": [\n    {\n      \"name\": \"example-service\",\n      \"project_id\": 46267196,\n      \"regions\": [\n        \"europe-west1\",\n        \"us-east1\",\n        \"us-west1\"\n      ]\n    }\n  ]\n}\n\n```\n\n\nOnce provisioned, a deployment project and necessary infrastructure will be\ncreated for a service.\n\n\n### Configure\n\n\nAfter a service is provisioned, the next challenge is the configuration for\na service. In Runway,\n[Reconciler](https://gitlab.com/gitlab-com/gl-infra/platform/runway/runwayctl)\nis a component that is responsible for configuring and deploying services by\naligning the actual state with the desired state using Golang and Terraform.\n\n\nHere’s a simplified example of an application developer configuring GitLab\nCI/CD in their service project:\n\n\n```\n\n# .gitlab-ci.yml\n\nstages:\n  - validate\n  - runway_staging\n  - runway_production\n\ninclude:\n  - project: 'gitlab-com/gl-infra/platform/runway/runwayctl'\n    file: 'ci-tasks/service-project/runway.yml'\n    inputs:\n      runway_service_id: example-service\n      image: \"$CI_REGISTRY_IMAGE/${CI_PROJECT_NAME}:${CI_COMMIT_SHORT_SHA}\"\n      runway_version: v3.22.0\n\n# omitted for brevity\n\n```\n\n\nRunway provides sane default values for configuration that are based on our\nexperience in delivering stable and reliable features to customers.\nAdditionally, service owners can configure infrastructure using a service\nmanifest file hosted in a service project. The service manifest uses JSON\nSchema for validation. When building this functionality with GitLab, Runway\nleverages [Pages](https://docs.gitlab.com/ee/user/project/pages/) for schema\ndocumentation.\n\n\nTo deliver this part of the platform, Runway leverages [CI/CD\ntemplates](https://docs.gitlab.com/ee/development/cicd/templates.html),\n[Releases](https://docs.gitlab.com/ee/user/project/releases/), and\n[Container\nRegistry](https://docs.gitlab.com/ee/user/packages/container\\_registry/) for\nintegrating with service projects.\n\n\nHere’s a simplified example of a service manifest:\n\n\n```\n\n# .runway/runway-production.yml\n\napiVersion: runway/v1\n\nkind: RunwayService\n\nspec:\n container_port: 8181\n regions:\n   - us-east1\n   - us-west1\n   - europe-west1\n\n# omitted for brevity\n\n```\n\n\nFor multi-region services, Runway injects an environment variable into the\ncontainer instance runtime, e.g. RUNWAY\\_REGION, so application developers\nhave the context to make any downstream dependencies regionally-aware, e.g.\nVertex AI API.\n\n\nOnce configured, a service project will be integrated with a deployment\nproject.\n\n\n### Deploy\n\n\nAfter a service project is configured, the next challenge is deploying a\nservice. In Runway, Reconciler handles this by triggering a deployment job\nin the deployment project when an MR is merged to the main branch. When\nbuilding this functionality with GitLab, Runway leverages [Trigger\nPipelines](https://docs.gitlab.com/ee/ci/triggers/) and [Multi-Project\nPipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream\\_pipelines.html\\#multi-project-pipelines)\nto trigger jobs from service project to deployment project.\n\n\n![trigger jobs from service project to deployment\nproject](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098671612.png)\n\n\nOnce a pipeline is running in a deployment project, it will be deployed to\nan environment. By default, Runway will provision staging and production\nenvironments for all services. At this point, Reconciler will apply any\nTerraform resource changes for infrastructure. When building this\nfunctionality with GitLab, Runway leverages\n[Environments/Deployments](https://docs.gitlab.com/ee/ci/environments/) and\n[GitLab-managed Terraform\nstate](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform\\_state.html)\nfor each service.\n\n\n![Reconciler applies any Terraform resource changes for\ninfrastructure](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098671614.png)\n\n\nRunway provides default application metrics for services. Additionally,\ncustom metrics can be used by enabling a sidecar container with\nOpenTelemetry Collector configured to scrape Prometheus and remote write to\nMimir. By providing observability out of the box, Runway is able to bake\nmonitoring into CI/CD pipelines.\n\n\nExample scenarios include gradual rollouts for blue/green deployments,\npreventing promotions to production when staging is broken, or automatically\nrolling back to previous revision when elevated error rates occur in\nproduction.\n\n\n![Runway bakes monitoring into CI/CD\npipelines](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098671615.png)\n\n\nOnce deployed, environments will serve the latest revision of a service. At\nthis point, you should have a good understanding of some of the challenges\nthat will be encountered, and how to solve them with GitLab features.\n\n\n## Migrating to multi-region in production\n\n\nAfter extending Runway components to support multi-region in Cloud Run, the\nfinal challenge was migrating from AI Gateway’s single-region deployment in\nproduction with zero downtime. Today, teams using Runway to deploy their\nservices can self-serve on regions making a multi-region deployment just as\nsimple as a single-region deployment. \n\n\nWe were able to iterate on building multi-region functionality without\nimpacting existing infrastructure by using semantic versioning for Runway.\nNext, we’ll share some learnings from the migration that may inform how to\noperate services for an internal multi-region platform.\n\n\n### Dry run deployments\n\n\nIn Runway, Reconciler will apply Terraform changes in CI/CD. The trade-off\nis that plans cannot be verified in advance, which could risk inadvertently\ndestroying or misconfiguring production infrastructure. To solve this\nproblem, Runway will perform a “dry run” deployment for MRs.\n\n\n![\"Dry run\"\ndeployment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098671616.png)\n\n\nFor migrating AI Gateway, dry run deployments increased confidence and\nhelped mitigate risk of downtime during rollout. When building an internal\nplatform with GitLab, we recommend supporting dry run deployments from the\nstart.\n\n\n### Regional observability\n\n\nIn Runway, existing observability was aggregated by assuming a single-region\ndeployment. To solve this problem, Runway observability was retrofitted to\ninclude a new region label for Prometheus metrics.\n\n\nOnce metrics were retrofitted, we were able to introduce service level\nindicators (SLIs) for both regional Cloud Run services and global load\nbalancing. Here’s an example dashboard screenshot for a general Runway\nservice:\n\n\n![dashboard screenshot for a general Runway\nservice](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098671617.png)\n\n\n***Note:** Data is not actual production data and is only for illustration\npurposes.*\n\n\nAdditionally, we were able to update our service level objectives (SLOs) to\nsupport regions. As a result, service owners could be alerted when a\nspecific region experiences an elevated error rate, or increase in response\ntimes.\n\n\n![screenshot of\nalerts](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098672/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098671617.png)\n\n\n***Note:** Data is not actual production data and is only for illustration\npurposes.*\n\n\nFor migrating AI Gateway, regional observability increased confidence and\nhelped provide more visibility into new infrastructure. When building an\ninternal platform with GitLab, we recommend supporting regional\nobservability from the start.\n\n\n### Self-service regions\n\n\nThe Infrastructure department successfully performed the initial migration\nof multi-region support for AI Gateway in production with zero downtime.\nGiven the risk associated with rolling out a large infrastructure migration,\nit was important to ensure the service continued working as expected.\n\n\nShortly afterwards, service owners began self-serving additional regions to\nmeet the growth of customers. At the time of writing, [GitLab\nDuo](https://about.gitlab.com/gitlab-duo/) is available in six regions\naround the globe and counting. Service owners are able to configure the\ndesired regions, and Runway will provide guardrails along the way in a\nscalable solution.\n\n\nAdditionally, three other internal services have already started using\nmulti-region functionality on Runway. Application developers have entirely\nself-served functionality, which validates that we’ve provided a good\nplatform experience for service owners. For a platform play, a scalable\nsolution like Runway is considered a good outcome since the Infrastructure\ndepartment is no longer a blocker.\n\n\n## What’s next for Runway\n\n\nBased on how quickly we could iterate to provide results for customers, the\nSaaS Platforms department has continued to invest in Runway. We’ve grown the\nRunway team with additional contributors, started evolving the platform\nruntime (e.g. Google Kubernetes Engine), and continue dogfooding with\ntighter integration in the product.\n\n\nIf you’re interested in learning more, feel free to check out\n[https://gitlab.com/gitlab-com/gl-infra/platform/runway](https://gitlab.com/gitlab-com/gl-infra/platform/runway).\n\n\n## More Building GitLab with GitLab\n\n- [Why there is no MLOps without\nDevSecOps](https://about.gitlab.com/blog/there-is-no-mlops-without-devsecops/)\n\n- [Stress-testing Product\nAnalytics](https://about.gitlab.com/blog/building-gitlab-with-gitlab-stress-testing-product-analytics/)\n\n- [Web API Fuzz\nTesting](https://about.gitlab.com/blog/building-gitlab-with-gitlab-api-fuzzing-workflow/)\n\n- [How GitLab.com inspired\nDedicated](https://about.gitlab.com/blog/building-gitlab-with-gitlabcom-how-gitlab-inspired-dedicated/)\n\n- [Expanding our security certification\nportfolio](https://about.gitlab.com/blog/building-gitlab-with-gitlab-expanding-our-security-certification-portfolio/)\n",[108,788,787,1263,9,744,1264,742,851,720],"inside GitLab","google",{"slug":1266,"featured":90,"template":700},"building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features","content:en-us:blog:building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features.yml","Building Gitlab With Gitlab A Multi Region Service To Deliver Ai Features","en-us/blog/building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features.yml","en-us/blog/building-gitlab-with-gitlab-a-multi-region-service-to-deliver-ai-features",{"_path":1272,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1273,"content":1279,"config":1286,"_id":1288,"_type":13,"title":1289,"_source":15,"_file":1290,"_stem":1291,"_extension":18},"/en-us/blog/building-gitlab-with-gitlab-api-fuzzing-workflow",{"title":1274,"description":1275,"ogTitle":1274,"ogDescription":1275,"noIndex":6,"ogImage":1276,"ogUrl":1277,"ogSiteName":685,"ogType":686,"canonicalUrls":1277,"schema":1278},"Building GitLab with GitLab: Web API Fuzz Testing","Our new series shows how we dogfood new DevSecOps platform features to ready them for you. First up, security testing.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659740/Blog/Hero%20Images/building-gitlab-with-gitlab-no-type.png","https://about.gitlab.com/blog/building-gitlab-with-gitlab-api-fuzzing-workflow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Building GitLab with GitLab: Web API Fuzz Testing\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Mike Eddington\"},{\"@type\":\"Person\",\"name\":\"Eugene Lim\"}],\n        \"datePublished\": \"2023-05-09\",\n      }",{"title":1274,"description":1275,"authors":1280,"heroImage":1276,"date":1283,"body":1284,"category":785,"tags":1285},[1281,1282],"Mike Eddington","Eugene Lim","2023-05-09","At GitLab, we try to [dogfood\neverything](/handbook/product/product-processes/#dogfood-everything) to help\nus better understand the product, pain points, and configuration issues. We\nuse what we learn to build a more efficient, feature-rich platform and user\nexperience. In this first installment of our “Building GitLab with GitLab”\nseries, we will focus on security testing. We constantly strive to improve\nour security testing coverage and integrate it into our DevSecOps lifecycle.\nThese considerations formed the motivation for the API fuzzing dogfooding\nproject at GitLab. By sharing our lessons from building this workflow, we\nhope other teams can also learn how to integrate GitLab’s Web API Fuzz\nTesting and solve some common challenges.\n\n\n## What is Web API Fuzz Testing?\n\n\nWeb API Fuzz Testing involves generating and sending various unexpected\ninput parameters to a web API in an attempt to trigger unexpected behavior\nand errors in the API backend. By analyzing these errors, you can discover\nbugs and potential security issues missed by other scanners that focus on\nspecific vulnerabilities. GitLab's Web API Fuzz Testing complements and\nshould be run in addition to GitLab Secure’s other security scanners such as\nstatic application security testing\n([SAST](https://docs.gitlab.com/ee/user/application_security/sast/)) and\ndynamic application security testing\n([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)) APIs.\n\n\n## Auto-generating an OpenAPI specification\n\nTo run the Web API Fuzzing Analyzer, you need one of the following:\n\n* OpenAPI Specification - Version 2 or 3\n\n* GraphQL Schema\n\n* HTTP Archive (HAR)\n\n* Postman Collection - Version 2.0 or 2.1\n\n\nAt the start of the API fuzzing project, the [API Vision working\ngroup](/company/team/structure/working-groups/api-vision/) was also working\non an issue to automatically document [GitLab’s REST API endpoints in an\nOpenAPI specification](https://gitlab.com/groups/gitlab-org/-/epics/8636),\nso we worked with our colleague Andy Soiron on implementing it. Because\nGitLab uses the [grape](https://github.com/ruby-grape/grape) API framework,\nAndy had already identified and\n[tested](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95877) the\n[grape-swagger](https://github.com/ruby-grape/grape-swagger) gem that\nauto-generates an OpenAPI v2 specification based on existing grape\nannotations. For example, the following API endpoint code:\n\n\n```\n     Class.new(Grape::API) do\n       format :json\n       desc 'This gets something.'\n       get '/something' do\n         { bla: 'something' }\n       end\n       add_swagger_documentation\n     end\n``` \n\nWill be parsed by grape-swagger into:\n\n\n```\n\n{\n  // rest of OpenAPI v2 specification\n  …\n  \"paths\": {\n    \"/something\": {\n      \"get\": {\n        \"description\": \"This gets something.\",\n        \"produces\": [\n          \"application/json\"\n        ],\n        \"operationId\": \"getSomething\",\n        \"responses\": {\n          \"200\": {\n            \"description\": \"This gets something.\"\n          }\n        }\n      }\n    }\n  }\n}\n\n```\n\n\n\nHowever, with almost 2,000 API operations with different requirements and\nformats, a lot of additional work needed to be done to resolve edge cases\nthat did not meet the requirements of grape-swagger or the OpenAPI format.\nFor example, one simple case was API endpoints that accept file parameters,\nsuch as the [upload metric image\nendpoint](https://docs.gitlab.com/ee/api/issues.html#upload-metric-image).\nGitLab uses the\n[Workhorse](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse)\nsmart reverse proxy to handle \"large\" HTTP requests such as file uploads. As\nsuch, file parameters must be of the type WorkhorseFile:\n\n\n\n```\n\nnamespace ':id/issues/:issue_iid/metric_images' do\n            …\n            desc 'Upload a metric image for an issue' do\n              success Entities::IssuableMetricImage\n            end\n            params do\n              requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded'\n              optional :url, type: String, desc: 'The url to view more metric info'\n              optional :url_text, type: String, desc: 'A description of the image or URL'\n            end\n            post do\n              require_gitlab_workhorse!\n```\n\n\nBecause grape-swagger does not recognize what OpenAPI type WorkhorseFile\ncorresponds to, it excludes the parameter from its output. We fixed this by\nadding a grape-swagger-specific documentation to override the type during\ngeneration:\n\n\n```\n             requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded', documentation: { type: 'file' }\n```\n\n\nHowever, not all edge cases could be resolved with a simple\nmatch-and-replace in the grape annotations. For example, Ruby on Rails\nsupports wildcard segment parameters. A route like `get\n'books/*section/:title'` would\nmatch`books/some/section/last-words-a-memoir`. In addition, the URI would be\nparsed such that the `section` path parameter would have the value\n`some/section` and the `title` path parameter would have the value\n`last-words-a-memoir`.\n\n\nCurrently, grape-swagger does not recognize these wildcard segments as path\nparameters. For example, the route would generate:\n\n\n```\n\n\"paths\": {\n  \"/api/v2/books/*section/{title}\": {\n    \"get\": {\n    ...\n      \"parameters\": [\n         {\n           \"in\": \"query\", \"name\": \"*section\"\n           ...\n  }\n}\n\n```\n\n\nInstead of the expected:\n\n\n```\n\n\"paths\": {\n  \"/api/v2/books/{section}/{title}\": {\n    \"get\": {\n    ...\n      \"parameters\": [\n         {\n           \"in\": \"path\", \"name\": \"section\"\n           ...\n  }\n}\n\n```\n\n\nAs such, we also needed to make several patches to grape-swagger, which we\nforked while waiting for the changes to be accepted upstream. Nevertheless,\nwith lots of careful checking and cooperation across teams, we managed to\nget the OpenAPI specification generated for most of the endpoints.\n\n\n## Performance tuning\n\n\nWith the OpenAPI specification, we could now begin with the API fuzzing.\nGitLab already uses the [Review\nApps](https://docs.gitlab.com/ee/ci/review_apps/) feature to generate\ntesting environments for some feature changes, providing a readily available\nfuzzing target. However, given the large number of endpoints, it would be\nimpossible to expect a standard shared runner to complete fuzzing in a\nsingle job. The Web API Fuzz Testing documentation includes a [performance\ntuning\nsection](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#performance-tuning-and-testing-speed)\nthat recommends the following:\n\n\n* using a multi-CPU Runner\n\n* excluding slow operations\n\n* splitting a test into multiple jobs\n\n* excluding operations in feature branches, but not default branch\n\n\nThe first recommendation was easy to implement with a dedicated fuzzing\nrunner. We recommend doing this for large scheduled fuzzing workflows,\nespecially if you select the Long-100 fuzzing profile. We also began\nexcluding slow operations by checking the job logs for the time taken to\ncomplete each operation. Along the way, we identified other endpoints that\nneeded to be excluded, such as the [revoke token\nendpoint](https://docs.gitlab.com/ee/api/personal_access_tokens.html#revoke-a-personal-access-token)\nthat prematurely ended the fuzzing session.\n\n\nSplitting the test into multiple jobs took the most effort due to the\nrequirements of the OpenAPI format. Each OpenAPI document includes a\nrequired set of objects and fields, so it is not simply a matter of\nsplitting after a fixed number of lines. Additionally, each operation relies\non entities defined in the definitions object, so we needed to ensure that\nwhen splitting the OpenAPI specification, the entities required by the\nendpoints were included. We also wrote a quick script to fill the example\nparameter data with actual data from the testing environment, such as\nproject IDs.\n\n\nWhile it was possible to run these scripts locally, then push the split jobs\nand OpenAPI specifications to the repository, this created a large number of\nchanges every time we updated the original OpenAPI specification. Instead,\nwe adapted the workflow to use dynamically generated child pipelines that\nwould split the OpenAPI document in a CI job, then generate a child pipeline\nwith jobs for each split document. This made iterating a lot easier and more\nagile. We have uploaded [the scripts and pipeline\nconfiguration](https://gitlab.com/eugene_lim/api-fuzzing-dogfooding) for\nreference.\n\n\nBy tweaking the number of parallel jobs and fuzzing profile, we were\neventually able to achieve a reasonably comprehensive fuzzing session in an\nacceptable time frame. When tuning your own fuzzing workflow, balancing\nthese trade-offs is essential.\n\n\n## Triaging the API fuzzing findings\n\n\nWith the fuzzing done, we were now confronted with hundreds of findings.\nUnlike DAST analyzers that try to detect specific vulnerabilities, Web API\nFuzz Testing looks for unexpected behavior and errors that may not\nnecessarily be vulnerabilities. This is why fuzzing faults discovered by the\nAPI Fuzzing Analyzer show up as vulnerabilities with a severity of\n“Unknown.” This requires more involved triaging.\n\n\nFortunately, the Web API fuzzer also outputs Postman collections as\nartifacts in the Vulnerability Report page. These collections allow you to\nquickly repeat requests that triggered a fault during fuzzing. For this\nstage of the fuzzing workflow, we recommend that you set up a local instance\nof the application so that you can easily check logs and debug specific\nfaults. In this case, we ran the [GitLab Development\nKit](https://gitlab.com/gitlab-org/gitlab-development-kit).\n\n\nMany of the faults occurred due to a lack of error handling for unexpected\ninputs. We created issues from the Vulnerability Report page, and if we\nfound that a particular fault had the same root cause as a previously\ntriaged fault, we linked the vulnerability to the original issue instead.\n\n\n## Lessons learned\n\n\nThe API fuzzing dogfooding project turned out to be a fruitful exercise that\nbenefited other workstreams at GitLab, such as the API documentation\nproject. In addition, tuning and triaging helped us identify key pain points\nin the process for improvement. Automated API documentation generation is\ndifficult even with OpenAPI, particularly on a long-lived codebase. GitLab’s\nexisting annotations and tests helped speed up documentation via a\ndistributed, asynchronous workflow across multiple teams. In addition, many\nGitLab features such as Review Apps, Vulnerability Reports, and dynamically\ngenerated child pipelines helped us build a robust fuzzing workflow.\n\n\nThere are still many improvements that can be made to the workflow. Moving\nto OpenAPI v3 could improve endpoint coverage. The Secure team also wrote a\n[HAR Recorder](https://gitlab.com/gitlab-org/security-products/har-recorder)\ntool that could help generate HAR files on the fly instead of relying on\nstatic documentation. For now, due to the high compute cost of fuzzing\nthousands of operations in GitLab’s API, the workflow is better suited to a\nscheduled pipeline instead of GitLab’s core pipeline.\n\n\nFor teams that have already implemented several layers of static and dynamic\nchecks and want to take further steps to increase coverage, we recommend\ntrying a Web API fuzzing exercise as a way to validate assumptions and\ndiscover “unknown unknowns” in your code.\n\n\nWe encourage you to get familiar with API fuzzing and let us know how it\nworks for you. If you face any issues or have any feedback, please file an\nissue at the [issue tracker on\nGitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/). Use the\n`~\"Category:API Security\"` label when opening a new issue regarding API\nfuzzing to ensure it is quickly reviewed by the appropriate team members.\n",[1263,830,1263,1057,9],{"slug":1287,"featured":6,"template":700},"building-gitlab-with-gitlab-api-fuzzing-workflow","content:en-us:blog:building-gitlab-with-gitlab-api-fuzzing-workflow.yml","Building Gitlab With Gitlab Api Fuzzing Workflow","en-us/blog/building-gitlab-with-gitlab-api-fuzzing-workflow.yml","en-us/blog/building-gitlab-with-gitlab-api-fuzzing-workflow",{"_path":1293,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1294,"content":1300,"config":1307,"_id":1309,"_type":13,"title":1310,"_source":15,"_file":1311,"_stem":1312,"_extension":18},"/en-us/blog/can-chatgpt-resolve-gitlab-issues",{"title":1295,"description":1296,"ogTitle":1295,"ogDescription":1296,"noIndex":6,"ogImage":1297,"ogUrl":1298,"ogSiteName":685,"ogType":686,"canonicalUrls":1298,"schema":1299},"Testing ChatGPT: Can it solve a GitLab issue?","We put ChatGPT to the test to see if it could contribute to GitLab. Here's what we learned.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749670171/Blog/Hero%20Images/akshay-nanavati-Zq6HerrBPEs-unsplash.jpg","https://about.gitlab.com/blog/can-chatgpt-resolve-gitlab-issues","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Testing ChatGPT: Can it solve a GitLab issue?\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"John Coghlan\"},{\"@type\":\"Person\",\"name\":\"Fatima Sarah Khalid\"}],\n        \"datePublished\": \"2022-12-15\",\n      }",{"title":1295,"description":1296,"authors":1301,"heroImage":1297,"date":1303,"body":1304,"category":785,"tags":1305},[1302,1217],"John Coghlan","2022-12-15","\nChatGPT has taken the tech world by storm since its [launch on November 30](https://openai.com/blog/chatgpt/). Media coverage, front page posts on Hacker News, Twitter threads, and videos - everywhere you look, there is another story.\n\nThe [GitLab Slack](https://handbook.gitlab.com/handbook/communication/#slack) was no different. In threads across Slack channels, including those for developer evangelism, UX, the CEO, random news, and every space in between, our team was chatting about this exciting new tool.\n\nAs we got more familiar with the tool, we started to learn about numerous things it can do. Here are a few that we found:\n\n- It can write poetry about GitLab features.\n- It can write blog posts.\n- It can write unit tests.\n- It gives advice on how to use certain features of GitLab.\n- It conducts competitive analysis.\n\nThere’s quite a bit more out there, including [inventing a new language](https://maximumeffort.substack.com/p/i-taught-chatgpt-to-invent-a-language) and [building a virtual machine](https://www.engraved.blog/building-a-virtual-machine-inside/). We can’t recall any technology that has generated more excitement in such a short time.\n\nWe acknowledge there are ethical and licensing concerns around using AI-generated code. For the purpose of this blog post, we will focus strictly on the capabilities of ChatGPT.\n\n## Testing ChatGPT\n\nAs members of GitLab’s [Developer Relations team](/handbook/marketing/developer-relations/), where we’re focused on growing our community of contributors and evangelists, our first reaction was to think of how this tool can help our contributors. The responses to questions like “How can I get started contributing to GitLab?” were cool but didn’t move the needle. So then we asked ourselves: Can we use ChatGPT to make a contribution to GitLab?\n\nHaving already been testing the tool, we knew we’d need to look for a very specific type of issue. We started to fine-tune our search. Here are the steps we took to find a potential issue:\n\n- Visited [https://gitlab.com/gitlab-org](https://gitlab.com/gitlab-org) and pulled up all the open issues by clicking on `Issues` in the left side nav.\n- Searched for all issues with the [“good for new contributors” label](https://gitlab.com/gitlab-org/gitlab/-/issues/?sort=updated_desc&state=opened&label_name%5B%5D=good%20for%20new%20contributors&first_page_size=20). This returned 482 issues at the time of writing.\n- Set [`“Assignee = None”`](https://gitlab.com/gitlab-org/gitlab/-/issues/?sort=updated_desc&state=opened&label_name%5B%5D=good%20for%20new%20contributors&assignee_id=None&first_page_size=20) to filter out issues that have already been assigned to someone. This trimmed things down even more.\n\nAfter poking through a few issues, this one caught our attention: an issue that is a request to [de-depulicate identical dropdown functionality from two Vue components into a shared component](https://gitlab.com/gitlab-org/gitlab/-/issues/221293).\n\n### Getting started\n\nWe opened up a ChatGPT thread via [https://chat.openai.com/chat](https://chat.openai.com/chat). If you don’t already have an account, you’ll need to register with OpenAI.\n\nThen we started a thread with Assistant (that’s what is returned when you ask ChatGPT what is its name). We asked Assistant to create a shared Vue component called `SharedDropdownFilter` from the duplicate code in `GroupsDropdownFilter` and `ProjectsDropdownFilter` and passed the scripts of both components.\n\nChatGPT replied with:\n\n>You can create a new `SharedDropdownFilter` component that contains the duplicate code and use it in both `GroupsDropdownFilter` and `ProjectsDropdownFilter`:\n\nand provided a template to describe the use. The template provided wasn’t quite what we were looking for.\n\n```vue\n\u003Ctemplate>\n  \u003CGlDropdown\n    :aria-label=\"label\"\n    :class=\"{ disabled: loading }\"\n    :disabled=\"loading\"\n    :selected=\"selectedProjectsLabel\"\n    :variant=\"multiSelect ? 'inline' : 'single-line'\"\n  >\n    \u003Ctemplate v-slot:menu>\n      \u003Cdiv class=\"py-4\">\n        \u003CGlSearchBoxByType\n          autofocus\n          class=\"search-box\"\n          :placeholder=\"searchPlaceholder\"\n          :type=\"searchType\"\n          v-model=\"searchTerm\"\n        />\n\n        \u003CGlDropdownHeader\n          v-if=\"selectedProjects.length && multiSelect\"\n          :class=\"{ 'd-none': !searchTerm }\"\n          class=\"js-dropdown-header\"\n        >\n          \u003Cspan>\n            {{ selectedProjectsLabel }}\n          \u003C/span>\n\n          \u003Cbutton\n```\n\nSo we tried again, this time being more specific and asking ChatGPT to “Create a shared Vue component that combines the two components `ProjectsDropdownFilter` and  `GroupsDropdownFilter`”. We passed in the full component files, including the templates.\n\nChatGPT responded with instructions, a sample Vue component, and an explanation of the functionality:\n\n> Here is a sample implementation of a shared Vue component that combines the two components `ProjectsDropdownFilter` and `GroupsDropdownFilter`:\n\n```vue\n\u003Ctemplate>\n  \u003Cdiv>\n    \u003CProjectsDropdownFilter\n      :groupId=\"groupId\"\n      :multiSelect=\"multiSelect\"\n      :label=\"label\"\n      :queryParams=\"queryParams\"\n      :defaultProjects=\"defaultProjects\"\n      @selected=\"onProjectsSelected\"\n    />\n    \u003CGroupsDropdownFilter\n      :multiSelect=\"multiSelect\"\n      :label=\"label\"\n      :queryParams=\"queryParams\"\n      :defaultGroups=\"defaultGroups\"\n      @selected=\"onGroupsSelected\"\n    />\n  \u003C/div>\n\u003C/template>\n\n\u003Cscript>\nimport { ProjectsDropdownFilter } from './ProjectsDropdownFilter.vue';\nimport { GroupsDropdownFilter } from './GroupsDropdownFilter.vue';\n\nexport default {\n  name: 'SharedDropdownFilter',\n  components: {\n    ProjectsDropdownFilter,\n    GroupsDropdownFilter,\n  },\n  props: {\n    groupId: {\n      type: Number,\n      required: true,\n    },\n    multiSelect: {\n      type: Boolean,\n      required: false,\n      default: false,\n    },\n    label: {\n      type: String,\n      required: false,\n      default: 'Dropdown filter',\n    },\n    queryParams: {\n      type: Object,\n      required: false,\n      default: () => ({}),\n    },\n    defaultProjects: {\n      type: Array,\n      required: false,\n      default: () => [],\n    },\n    defaultGroups: {\n      type: Array,\n      required: false,\n      default: () => [],\n    },\n  },\n  data() {\n    return {\n      selectedProjects: this.defaultProjects || [],\n      selectedGroups: this.defaultGroups || [],\n    };\n  },\n  methods: {\n    onProjectsSelected(selectedProjects) {\n      this.selectedProjects = selectedProjects;\n      this.$emit('selected', this.selectedProjects, this.selectedGroups);\n    },\n    onGroupsSelected(selectedGroups) {\n      this.selectedGroups = selectedGroups;\n      this.$emit('selected', this.selectedProjects, this.selectedGroups);\n    },\n  },\n};\n\u003C/script>\n```\n\n> In this implementation, the `SharedDropdownFilter` component is rendered as a container for the `ProjectsDropdownFilter` and` GroupsDropdownFilter` components. It passes down the `groupId`, `multiSelect`, `label`, `queryParams`, and `defaultProjects` props to the `ProjectsDropdownFilter` component, and the `multiSelect`, `label`, `queryParams`, and `defaultGroups` props to the `GroupsDropdownFilter` component. It also listens to the selected event emitted by the two components and updates its own `selectedProjects` and `selectedGroups` data accordingly. The `SharedDropdownFilter` component emits its own `selected` event with the updated `selectedProjects` and `selectedGroups` data when it receives the selected event from the `ProjectsDropdownFilter` and `GroupsDropdownFilter` components. This allows the parent component of the `SharedDropdownFilter` to be notified whenever the selected projects and groups are updated, so that it can take appropriate action, such as updating the data it displays.\n\nWith ChatGPT’s shared Vue component, we could start an MR for the issue, but would need to manually review the code and run tests in order to make sure the deduplication was done correctly.\n\n### Adjusting the filters\n\nIn hopes of finding an issue that ChatGPT could fully solve, this time we filtered using two labels (`good for new contributors` and `quick win`), Assignee = None, and added the keyword “refactor” to look for issues that required less net new code. At the time of writing, this [query](https://gitlab.com/gitlab-org/gitlab/-/issues/?search=refactor&sort=updated_asc&state=opened&label_name%5B%5D=quick%20win&label_name%5B%5D=good%20for%20new%20contributors&assignee_id=None&first_page_size=20) returned one issue: [a refactor for widget state tests](https://gitlab.com/gitlab-org/gitlab/-/issues/228611).\n\nAfter reviewing the issue, we opened the `/spec/frontend/vue_merge_request_widget/stores/get_state_key_spec.js` file mentioned in the issue description by navigating to that file on GitLab.com SaaS and opening the dropdown on the blue button at the top of the file frame, selecting “Open in Web IDE”, and then clicking the button to open the file in GitLab’s built-in Web IDE.\n\nWe copied the code from the file and asked ChatGPT to refactor these tests into a table of `it.each`. It was interesting to watch the output as ChatGPT provided an example of how we could refactor the tests using `it.each`:\n\n```javascript\nimport getStateKey from '~/vue_merge_request_widget/stores/get_state_key';\n\ndescribe('getStateKey', () => {\n  const context = {\n    mergeStatus: 'checked',\n    autoMergeEnabled: false,\n    canMerge: true,\n    onlyAllowMergeIfPipelineSucceeds: false,\n    isPipelineFailed: false,\n    hasMergeableDiscussionsState: false,\n    isPipelineBlocked: false,\n    canBeMerged: false,\n    projectArchived: false,\n    branchMissing: false,\n    commitsCount: 2,\n    hasConflicts: false,\n    draft: false,\n    detailedMergeStatus: null,\n  };\n\n  it.each([\n    [{}, null],\n    [{detailedMergeStatus: 'MERGEABLE'}, 'readyToMerge'],\n    [{detailedMergeStatus: 'DISCUSSIONS_NOT_RESOLVED'}, 'unresolvedDiscussions'],\n    [{detailedMergeStatus: 'DRAFT_STATUS'}, 'draft'],\n    [{detailedMergeStatus: 'CI_MUST_PASS'}, 'pipelineFailed'],\n    [{shouldBeRebased: true}, 'rebase'],\n    [{hasConflicts: true}, 'conflicts'],\n    [{detailedMergeStatus: 'CHECKING'}, 'checking'],\n    [{commitsCount: 0}, 'nothingToMerge'],\n    [{commitsCount: 1, branchMissing: true}, 'missingBranch'],\n    [{projectArchived: true}, 'archived'],\n  ])('should return \"%s\" when called with %p', (expected, input) => {\n    const bound = getStateKey.bind(Object.assign({}, context, input));\n    expect(bound()).toEqual(expected);\n  });\n\n  it('returns rebased state key', () => {\n    const context = {\n      mergeStatus: 'checked',\n      autoMergeEnabled: false,\n      canMerge: true,\n      onlyAllowMergeIfPipelineSucceeds: true,\n      isPipelineFailed: true,\n      hasMergeableDiscussionsState: false,\n      isPipelineBlocked: false,\n      canBeMerged: false,\n      shouldBeRebased: true,\n      projectArchived: false,\n      branchMissing: false,\n      commitsCount: 2,\n      hasConflicts: false,\n      draft: false,\n    };\n    const bound = getStateKey.bind(context);\n\n    expect(bound()).toEqual('rebase');\n  });\n});\n\n\n```\n\nAlong with an explanation of `it.each`:\n\n> Note that `it.each` is a utility provided by Jest that allows you to write the same test with different inputs and expected results. This can make your tests more concise and easier to read.\n\nOur next steps were to:\n- Paste the output from ChatGPT into the Web IDE.\n- Click the blue “Commit” button to commit the code.\n- Name the branch.\n- Open a merge request.\n\nAll of this can be done inside [GitLab’s Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/) without leaving the browser.\n\n### Examining the results\n\nOnce we opened the merge request, a CI/CD pipeline was created to test the change. Unfortunately, the pipeline jobs for the Jest tests failed because the tests were returning `null` for each case instead of the expected results. We went back to ChatGPT to see if it could help us understand why the test was returning null for all cases. ChatGPT suggested a few different implementations to check to make sure things are defined currently:\n\n> There are a few reasons why the tests in the `it.each` function might be returning null for every case. Here are a few possibilities:\n>\n> If the `getStateKey` function is not implemented correctly, it might always return null, regardless of the input.\n> If the `getStateKey` function relies on certain variables that are not defined in the test cases, it might return null because those variables are undefined.\n> If the context object passed to the `getStateKey` function is not being merged correctly with the input object in the test cases, the function might not have access to all of the necessary information, and it might return null.\n\nWhile ChatGPT couldn’t solve the issue with the returned value directly, it provided a lot of possibilities to look into and offered suggestions of what variables or functions to debug. The suggestions prompted us to add code changes that logged the variable values onto the [JavaScript debug console](https://www.w3schools.com/js/js_debugging.asp), find the implementation of the `getStateKey` function, and figure out why the return value was null. Whenever we came across something unfamiliar in the code, like syntax in the `it.each` that wasn’t familiar, we asked ChatGPT for clarification or a helpful example. Many times throughout this experiment, working with ChatGPT felt like “rubber duck debugging,” but with an AI with which you have to be very specific about your ask.\n\n## What we learned from ChatGPT\n\nIn the end, we weren’t able to figure out why our tests were returning null, so we asked the front-end team if someone could review the code. Senior Frontend Engineer [Angelo Gulina](https://gitlab.com/agulina) reviewed the MR. He found that the solution was actually quite trivial: The order of parameters was inverted, resulting in a comparison that led to null! In his assessment, ChatGPT wasn’t able to provide a working solution, but would be able to provide solutions and ideas to an engineer with some experience with the codebase. It delivered a clean, organized solution and answered the task of combining the tests into an it.each table. It could not, however, catch the actual error (the inversion of parameters) or correctly guess why the tests were returning null.\n\nLet's circle back to the question that started this experiment: Can we use ChatGPT to contribute to GitLab? At this time, we’d say, \"yes,\" and you will need some understanding of the code to complete your solution. Since ChatGPT is a language model trained by OpenAI, it can only answer questions and provide information addressed in the model, which means answers requiring contextual specificity may fall short of what is needed to resolve an issue. However, it’s a tool that can help you if you’re stuck, need more clarification on a code snippet, or are trying to refactor some code. It was fascinating for us to experiment with ChatGPT and we were excited to see what it was capable of. The code provided, however, lacked some of the valuable insight and industry experience that a community of contributors can provide.\n\nAt GitLab, our [community and our open source stewardship](https://about.gitlab.com/company/strategy/#dual-flywheels) are part of our company strategy. Thousands of open source contributors worldwide have helped make GitLab what it is today. We see potential for ChatGPT and similar AI tools, not as a replacement for our community, but a way to make our community more efficient and enable more people to contribute GitLab.\n\n\n\n\n",[9,268,1306,789],"contributors",{"slug":1308,"featured":6,"template":700},"can-chatgpt-resolve-gitlab-issues","content:en-us:blog:can-chatgpt-resolve-gitlab-issues.yml","Can Chatgpt Resolve Gitlab Issues","en-us/blog/can-chatgpt-resolve-gitlab-issues.yml","en-us/blog/can-chatgpt-resolve-gitlab-issues",{"_path":1314,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1315,"content":1321,"config":1327,"_id":1329,"_type":13,"title":1330,"_source":15,"_file":1331,"_stem":1332,"_extension":18},"/en-us/blog/cascading-merge-requests-with-gitlab-flow",{"title":1316,"description":1317,"ogTitle":1316,"ogDescription":1317,"noIndex":6,"ogImage":1318,"ogUrl":1319,"ogSiteName":685,"ogType":686,"canonicalUrls":1319,"schema":1320},"How to adopt a cascading merge request strategy with GitLab Flow","This tutorial explains how to consolidate updates in a single branch and propagate them to other branches using ucascade bot.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679851/Blog/Hero%20Images/cascade.jpg","https://about.gitlab.com/blog/cascading-merge-requests-with-gitlab-flow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to adopt a cascading merge request strategy with GitLab Flow\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Madou Coulibaly\"}],\n        \"datePublished\": \"2023-08-31\",\n      }",{"title":1316,"description":1317,"authors":1322,"heroImage":1318,"date":1324,"body":1325,"category":785,"tags":1326},[1323],"Madou Coulibaly","2023-08-31","Git offers a range of branching strategies and workflows that can be\nutilized to enhance organization, efficiency, and code quality. Employing a\nwell-defined workflow helps foster a successful and streamlined development\nprocess. By implementing the [release branches using GitLab\nFlow](https://docs.gitlab.com/ee/topics/gitlab_flow.html#release-branches-with-gitlab-flow),\nyou can effectively handle multiple product releases. However, when it comes\nto fixing bugs, it often becomes necessary to apply the fix across various\nstable branches such as `main`,  `stable-1.0`, `stable-1.1`, and\n`stable-2.0`. The process of applying the fix to multiple locations can be\ntime-consuming, as it involves the manual creation of multiple merge\nrequests.\n\n\nBy consolidating updates in a single branch and propagating them to other\nbranches, the cascading merge approach establishes a central source of\ntruth, reducing confusion and maintaining consistency. In this blogpost, we\nwill guide you through setting up this approach for your GitLab project\nusing [ucascade bot](https://github.com/unblu/ucascade).\n\n\n## Getting started\n\nTo get started, you'll need the following prerequisites:\n\n\n### Environment\n  - a GitLab project that implemented [Release Branches Strategy](https://docs.gitlab.com/ee/topics/gitlab_flow.html#release-branches-with-gitlab-flow)\n  - a Kubernetes cluster\n\n### CLI\n  - git\n  - kubectl\n  - docker\n\n### Project access tokens\n\nFollow the instructions on the [Project access tokens\npage](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#create-a-project-access-token)\nto create two project access tokens –`ucascade` and `ucascade-approver` –\nwith the API scope in your GitLab project.\n\n\n![project access\ntokens](https://about.gitlab.com/images/blogimages/2023-06-22-cascading-merge-requests-with-gitlab-flow/pat.png){:\n.shadow.medium}\n\n\n## Deploy ucascade bot on Kubernetes\n\nFirst, create the `bots-fleet` namespace on Kubernetes.\n\n\n```\n\nkubectl create namespace bots-fleet\n\n```\n\n\nThen, create the `cascading-merge-secret` secret that contains the GitLab\nproject access tokens created previously.\n\n\n```\n\nkubectl create secret generic cascading-merge-secret -n bots-fleet \\\n\n--from-literal=gitlab-host=https://gitlab.com \\\n\n--from-literal=gitlab-api-token=\u003CUCASCADE_PROJECT_ACCESS_TOKEN> \\\n\n--from-literal=gitlab-api-token-approver=\u003CAPPROVER_BOT_PROJECT_ACCESS_TOKEN>\n\n```\n\n\nOnce done, (fork and) clone the [Cascading Merge\nrepository](https://gitlab.com/madou-stories/bots-fleet/cascading-merge)\nthat contains the Kubernetes manifests for the bot and replace the `host`\nfield in the `kube/ingress.yaml` file according to your Kubernetes domain.\n\n\n```yaml\n\napiVersion: networking.k8s.io/v1\n\nkind: Ingress\n\nmetadata:\n  annotations:\n    kubernetes.io/ingress.class: nginx\n  name: ucascade\n  namespace: bots-fleet\nspec:\n  rules:\n  - host: ucascade.\u003CKUBERNETES_BASED_DOMAIN>\n    http:\n      paths:\n      - backend:\n          service:\n            name: ucascade\n            port:\n              number: 80\n        path: /\n        pathType: Prefix\n\n``` \n\n\nNow, you are ready to deploy the `ucascade` bot.\n\n\n```\n\nkubectl apply -f kube/\n\n```\n\n\nYou should see the following resources deployed on Kubernetes:\n\n\n![ucascade-k8s](https://about.gitlab.com/images/blogimages/2023-06-22-cascading-merge-requests-with-gitlab-flow/ucascade-k8s.png){:\n.shadow.medium}\n\n\n**Note:** The `ucascade` image is based on the\n[ucascade-bot](https://github.com/unblu/ucascade-bot) and is located in the\n[Container\nRegistry](https://gitlab.com/madou-stories/bots-fleet/cascading-merge/container_registry)\nof the Cascading Merge repository.\n\n{: .note}\n\n\n## Create a GitLab webhook\n\nFollow the instructions on [the Webhooks\npage](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab)\nto create a webhook with the following variables: \n  - **URL**: `\u003CUCASCADE_INGRESS_URL>/ucascade/merge-request`\n  - **Trigger**: `Merge request events`\n\n![webhook](https://about.gitlab.com/images/blogimages/2023-06-22-cascading-merge-requests-with-gitlab-flow/webhook.png){:\n.shadow.medium}\n\n\n## Configure your Cascading Merge rule\n\nCreate a file called ucascade.json at the root level of your GitLab project\nas defined in [configuration\nfile](https://unblu.github.io/ucascade/tech-docs/11_ucascade-configuration-file.html#_configuration_file)\nand matched with your release definition.\n\n\n![configuration](https://about.gitlab.com/images/blogimages/2023-06-22-cascading-merge-requests-with-gitlab-flow/configuration.png){:\n.shadow.medium}\n\n\n## Testing the Cascading Merge\n\nNow create a branch and an MR from your default branch, make a change, and\nmerge it. The ucascade bot will propagate the change to all other release\nbranches by automatically creating cascading MRs. The following video\ndemonstrates the process:\n\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Ej7xf8axWMs\" title=\"Cascading Merge Approach\"\n  frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\n# Additional resources\n\nFind more information about the `ucascade` bot in the [ucascade\ndocumentation](https://unblu.github.io/ucascade/index.html).\n\n\n_Special thank you to Jérémie Bresson for authoring and open sourcing this\namazing bot!_\n",[108,809,742,9],{"slug":1328,"featured":90,"template":700},"cascading-merge-requests-with-gitlab-flow","content:en-us:blog:cascading-merge-requests-with-gitlab-flow.yml","Cascading Merge Requests With Gitlab Flow","en-us/blog/cascading-merge-requests-with-gitlab-flow.yml","en-us/blog/cascading-merge-requests-with-gitlab-flow",{"_path":1334,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1335,"content":1341,"config":1347,"_id":1349,"_type":13,"title":1350,"_source":15,"_file":1351,"_stem":1352,"_extension":18},"/en-us/blog/chat-about-your-merge-request-with-gitlab-duo",{"title":1336,"description":1337,"ogTitle":1336,"ogDescription":1337,"noIndex":6,"ogImage":1338,"ogUrl":1339,"ogSiteName":685,"ogType":686,"canonicalUrls":1339,"schema":1340},"Chat about your merge request with GitLab Duo","Learn how to use AI-powered Chat to quickly understand complex merge requests by asking about implementation choices, potential risks, and architectural decisions.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675536/Blog/Hero%20Images/blog-image-template-1800x945__2_.png","https://about.gitlab.com/blog/chat-about-your-merge-request-with-gitlab-duo","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Chat about your merge request with GitLab Duo\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Torsten Linz\"}],\n        \"datePublished\": \"2024-11-22\",\n      }",{"title":1336,"description":1337,"authors":1342,"heroImage":1338,"date":1344,"body":1345,"category":718,"tags":1346},[1343],"Torsten Linz","2024-11-22","Managing a merge request (MR) is an integral part of collaborative development, involving navigating through code changes, discussions, and dependencies to ensure high-quality outcomes. Whether you’re reviewing someone else’s code or trying to make your own changes clearer, the new [GitLab Duo Chat](https://about.gitlab.com/gitlab-duo/) capability, available in GitLab Duo Enterprise, can help simplify your workflow. Now, you can have a conversation with GitLab Duo Chat about an MR, directly inside GitLab.\n\n## What GitLab Duo Chat brings to an MR workflow\n\nImagine jumping into a merge request titled \"Add logging to order processing.\" Your goal is to onboard yourself to the MR as quickly as possible and to review it. You can use GitLab Duo Chat to onboard yourself faster and understand critical questions to accelerate your review:\n\n* \"Do the logs cover all failure scenarios, or are there any gaps where an issue might not be traceable?\"  \n* “Are there any potential privacy concerns with the logged data?\"  \n* \"Why was logging added at these specific points in the order processing workflow, and how does it help with debugging or monitoring?\"\n\n![MR context example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675670/Blog/Content%20Images/MR_Context_example.png)\n\nThese are the kinds of questions that GitLab Duo Chat is ready to answer – questions that let you quickly understand the intentions behind the changes and uncover any potential risks before diving into the details. Instead of spending a lot of time trying to follow code paths or waiting on the author to reply to your questions, you can start getting answers right away, saving valuable time.\n\n## In-depth conversations about MRs\n\nThe magic of this new chat capability isn’t just in summarizing code – it’s in its ability to support in-depth conversations about the MR at hand. Let's assume the logging MR also includes notifications and refactoring. You can ask specific, insightful questions, such as:\n\n* “What are the potential network failure points introduced by refactoring the payment service into a microservice?”  \n* \"Were there any trade-offs made in terms of consistency or accuracy for better performance?\"  \n* \"How are failures in sending notifications handled? Are retries implemented?\"\n\nInstead of simply telling you what changes have been made, GitLab Duo Chat helps you understand *why* those changes were made, what risks are involved, and how to mitigate them. It lets you dig deep and explore the context behind every line of code, every architectural decision, and every change in behavior within the specific MR you are working on.\n\nAnd it doesn't end with that one answer. You can engage in a follow-up conversation to dig deeper or to explore. \n\n## An evolving conversation tool\n\nWe’re really excited about how GitLab Duo Chat is evolving to become a true conversational partner for MR authors and reviewers alike. GitLab Duo Chat is [aware of the MR description, discussions, the code diff, and metadata of a single MR](https://docs.gitlab.com/ee/user/gitlab_duo_chat/index.html#the-context-chat-is-aware-of). It’s like having an assistant who is well-versed in your MR and ready to explain any part of it – or even rewrite parts, if that’s what you need.\n\nWith GitLab Duo Chat, onboarding yourself to a complex MR or understanding a change in-depth is faster and more intuitive than ever before.\n\n## We need your feedback\n\nWe’re eager to hear how GitLab Duo Chat works for you. All feedback helps us refine this feature and make it even more useful. Please share your experiences by commenting on our [issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues/464587). Please include the questions you asked, the response you got, and whether it helped you move forward. Together, we can make GitLab Duo Chat an indispensable tool for every merge request!\n\nFor a deeper dive into how to use GitLab Duo Chat, check out our [documentation](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples#ask-about-a-specific-merge-request) or watch our introductory video below. Start your first conversation today and let us know what you think!\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/4muvSFuWWL4?si=7W4mHWw2iUOzoTUz\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->  \n\n> Sample this new capability with [a free trial of GitLab Ultimate and GitLab Duo Enterprise](https://gitlab.com/-/trials/new).\n\n## Learn more about GitLab Duo Chat\n\n- [GitLab Duo Chat: Get to know productivity-boosting AI enhancements](https://about.gitlab.com/blog/gitlab-duo-chat-get-to-know-productivity-boosting-ai-enhancements/)\n- [GitLab Duo Chat, your at-the-ready AI assistant, is now generally available](https://about.gitlab.com/blog/gitlab-duo-chat-now-generally-available/)\n- [GitLab Duo Chat 101: Get more done on GitLab with our AI assistant](https://about.gitlab.com/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant/)",[720,495,696,9,694,956],{"slug":1348,"featured":6,"template":700},"chat-about-your-merge-request-with-gitlab-duo","content:en-us:blog:chat-about-your-merge-request-with-gitlab-duo.yml","Chat About Your Merge Request With Gitlab Duo","en-us/blog/chat-about-your-merge-request-with-gitlab-duo.yml","en-us/blog/chat-about-your-merge-request-with-gitlab-duo",{"_path":1354,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1355,"content":1361,"config":1368,"_id":1370,"_type":13,"title":1371,"_source":15,"_file":1372,"_stem":1373,"_extension":18},"/en-us/blog/ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups",{"title":1356,"description":1357,"ogTitle":1356,"ogDescription":1357,"noIndex":6,"ogImage":1358,"ogUrl":1359,"ogSiteName":685,"ogType":686,"canonicalUrls":1359,"schema":1360},"CI/CD automation: Maximize 'deploy freeze' impact across GitLab groups","Learn the benefits of managing deploy freezes at the group level and follow step-by-step guidance on implementation.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg","https://about.gitlab.com/blog/ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"CI/CD automation: Maximize 'deploy freeze' impact across GitLab groups\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Christian Nnachi\"}],\n        \"datePublished\": \"2024-02-08\",\n      }",{"title":1356,"description":1357,"authors":1362,"heroImage":1358,"date":1364,"body":1365,"category":1119,"tags":1366},[1363],"Christian Nnachi","2024-02-08","In the dynamic landscape of continuous integration and continuous deployment ([CI/CD](https://about.gitlab.com/topics/ci-cd/)), maintaining system stability during critical periods such as holidays, product launches, or maintenance windows can be challenging. Introducing new code during peak activity times raises the risk of issues affecting user experience. To strike a balance between innovation and stability, organizations may require a group-level deploy freeze — a strategic pause in deploying new code changes across groups to certain branches or environments.\n\n**Given that GitLab can be used for both continuous integration and continuous deployment efforts, GitLab's [Deploy Freeze](https://docs.gitlab.com/ee/user/project/releases/index.html#prevent-unintentional-releases-by-setting-a-deploy-freeze)** capability aims to address this exact need.\n\nScoped at the project level, deploy freezes can prevent unintended production releases during a period of time you specify by setting a deploy freeze period. Deploy freezes help reduce uncertainty and risk when continuously deploying changes for a single project.\n\nMost teams, however, do not have a single project that represents all of their production environment. Given that deploy freezes are set at the project level, managing and enforcing deploy freezes across many projects can be an arduous and error-prone task, leading to unpredictability and disruption. The need for an automated cross-project solution to ensure stability is obvious.\n\n## What is a group deploy freeze?\n\nThe [Group Deploy Freeze project](https://gitlab.com/cnnachi-demo/freezeperiods) takes the concept of individual project deploy freezes to the next level. It enables you to enforce the same deployment restrictions across one or many projects within a GitLab group from the GitLab UI.\n\nWhether you're managing a large suite of microservices or a collection of related projects, a group-managed deploy freeze solution provides a centralized mechanism to maintain stability.\n\n### Benefits of group deploy freeze\n\n**1. Centralized control**\n\nAdherence to your deployment strategy by allowing you to manage deploy freezes for multiple projects from a single location. This simplifies the process and reduces human errors.\n\n**2. Group-wide synchronization**\n\nEnforcing deploy freezes across an entire GitLab group ensures that all projects receive the same schedule at the same time. This maintains uniformity across your projects.\n\n**3. Streamlined collaboration**\n\nVisibility of changes to your development and operations teams can align their efforts effectively.\n\n## How to use GitLab Group Deploy Freeze\n\nWith [Group Deploy Freeze](https://gitlab.com/demos/solutions/group-deploy-freeze), GitLab CI becomes a general-purpose automation tool for ops-related changes, like setting deploy freezes on many projects.\n\nIn the following steps, you will successfully set up the Group Deploy Freeze feature. Remember to test thoroughly and consider any specific nuances of your team's deployment process.\n\n### Prerequisites\n\n- **GitLab account -** You need an active GitLab account with the necessary permissions to access and manage the projects within the target GitLab group.\n- **GitLab Personal Access Token (PAT) -** Generate a GitLab PAT with the permissions to read and write to the projects within the target GitLab group via the GitLab API. This token will be used by the Python script to authenticate API requests.\n- **Python environment -** Ensure that you have a Python environment set up on your machine or the environment where you plan to run the Python script. The script is written in Python, so you need a compatible Python interpreter.\n- **Python libraries -** Install the required Python libraries used by the script. These include requests, envparse, and python-gitlab. You can use pip to install these libraries.\n- **GitLab Group details -** Identify the GitLab group for which you want to manage deploy freezes. You'll need the group's slug (path) to specify which group the script will operate on.\n- **Time zone selection -** Decide on the time zone in which you want to schedule the deploy freezes. The time zone selection ensures that freeze periods are accurately timed based on your organization's preferred time zone.\n\n### Getting started\n\nTo use GitLab CI to author and automate the process of batch updating deploy freezes for all projects, fork the [Deploy Freeze project](https://gitlab.com/cnnachi-demo/freezeperiods), which will then create a CI/CD pipeline that iterates through your projects and applies the desired deploy freeze schedule. You can customize this project to fit your organization's workflow.\n\nThe provided project contains a `.gitlab-ci.yml` file and a Python script designed to automate the management of deploy freezes for multiple projects within a GitLab group. It uses the GitLab API and various Python libraries to create and delete deploy freeze periods, and is designed to be run as part of a CI/CD pipeline to ensure code stability during deployments within a GitLab group.\n\n### Commit and push changes\n\nCommit and push the changes to your repository to trigger the CI/CD pipeline.\n\n### Pipeline execution\n\n- In the [Group Deploy Freeze project](https://gitlab.com/demos/solutions/group-deploy-freeze) on the GitLab UI, go to Pipelines.\n- Select the \"Run Pipeline\" option on the top right corner of the page.\n- You should see the variables defined in the `.gitlab-ci.yml` file like:\n![Set variables](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676891/Blog/Content%20Images/Screenshot-2023-09-06-at-12-08-48-PM.png)\n- Define the values of the variables `FREEZE_START`, `FREEZE_END`, `CRON_TIME_ZONE` and `GROUP_SLUG`, then run the pipeline. You can define multiple freeze periods by skipping to the next line within the `FREEZE_START` and `FREEZE_END` variables.\n- Once the pipeline is successful, the freeze period should be populated in all projects within the defined groups.\n\n## Monitor and verify\n\n- Verify that these deploy freeze periods are being created and managed as intended.\n- Check your GitLab group's projects for deploy freezes during the specified periods.\n![Monitor and verify](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676891/Blog/Content%20Images/Screenshot-2023-09-12-at-2-08-24-PM.png)\n\n## Customization and iteration\n\n- If needed, iterate on the configuration, script, or pipeline based on your organization's requirements.\n- Make adjustments to freeze periods, time zones, project details, or other settings as needed.\n\nYou can optimize the group deploy feature by following the [Deploy freeze](https://docs.gitlab.com/ee/user/project/releases/index.html#prevent-unintentional-releases-by-setting-a-deploy-freeze) documentation, which outlines the steps to set up a `.freezedeployment` job that can conditionally block deployment jobs upon the presence of the `CI_DEPLOY_FREEZE` variable. By including the `.freezedeployment` template and extending it in your project's `.gitlab-ci.yml file`, you can prevent deployments during freeze periods, ensuring code stability. Manual deployment intervention is possible once the freeze period ends, allowing for controlled and predictable deployment processes across the group's projects.\n\n## Results\n\nBy extending deploy freezes to the group level, teams can easily streamline and enhance their deployment strategies to ensure consistency in preventing unintended production release during a period of time specified by you, whether it is a large company event or holiday. With the power of GitLab's API, CI/CD pipelines, and the flexibility of Python scripting, Group Deploy Freeze is your ally in maintaining code stability and predictability across diverse projects.\n\n> Get started with group deploy freezes today by visiting the [Group Deploy Freeze project](https://gitlab.com/cnnachi-demo/freezeperiods).",[108,694,1367,9],"production",{"slug":1369,"featured":6,"template":700},"ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups","content:en-us:blog:ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups.yml","Ci Cd Automation Maximize Deploy Freeze Impact Across Gitlab Groups","en-us/blog/ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups.yml","en-us/blog/ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups",{"_path":1375,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1376,"content":1382,"config":1389,"_id":1391,"_type":13,"title":1392,"_source":15,"_file":1393,"_stem":1394,"_extension":18},"/en-us/blog/ci-deployment-and-environments",{"title":1377,"description":1378,"ogTitle":1377,"ogDescription":1378,"noIndex":6,"ogImage":1379,"ogUrl":1380,"ogSiteName":685,"ogType":686,"canonicalUrls":1380,"schema":1381},"How to use GitLab CI to deploy to multiple environments","We walk you through different scenarios to demonstrate the versatility and power of GitLab CI.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662033/Blog/Hero%20Images/intro.jpg","https://about.gitlab.com/blog/ci-deployment-and-environments","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use GitLab CI to deploy to multiple environments\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ivan Nemytchenko\"},{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2021-02-05\",\n      }",{"title":1377,"description":1378,"authors":1383,"heroImage":1379,"date":1385,"body":1386,"category":785,"tags":1387,"updatedDate":1388},[1384,715],"Ivan Nemytchenko","2021-02-05","This post is a success story of one imaginary news portal, and you're the\nhappy\n\nowner, the editor, and the only developer. Luckily, you already host your\nproject\n\ncode on GitLab.com and know that you can\n\n[run tests with GitLab CI/CD](https://docs.gitlab.com/ee/ci/testing/).\n\nNow you’re curious if it can be [used for\ndeployment](/blog/how-to-keep-up-with-ci-cd-best-practices/), and how far\ncan you go with it.\n\n\nTo keep our story technology stack-agnostic, let's assume that the app is\njust a\n\nset of HTML files. No server-side code, no fancy JS assets compilation.\n\n\nDestination platform is also simplistic – we will use [Amazon\nS3](https://aws.amazon.com/s3/).\n\n\nThe goal of the article is not to give you a bunch of copy-pasteable\nsnippets.\n\nThe goal is to show the principles and features of [GitLab\nCI](/solutions/continuous-integration/) so that you can easily apply them to\nyour technology stack.\n\n{: .alert .alert-warning}\n\n\nLet’s start from the beginning. There's no continuous integration (CI) in\nour story yet.\n\n\n## At the starting line\n\n\n**Deployment**: In your case, it means that a bunch of HTML files should\nappear on your\n\nS3 bucket (which is already configured for\n\n[static website\nhosting](http://docs.aws.amazon.com/AmazonS3/latest/dev/HowDoIWebsiteConfiguration.html?shortFooter=true)).\n\n\nThere are a million ways to do it. We’ll use the\n\n[awscli](http://docs.aws.amazon.com/cli/latest/reference/s3/cp.html#examples)\nlibrary,\n\nprovided by Amazon.\n\n\nThe full command looks like this:\n\n\n```shell\n\naws s3 cp ./ s3://yourbucket/ --recursive --exclude \"*\" --include \"*.html\"\n\n```\n\n\n![Manual\ndeployment](https://about.gitlab.com/images/blogimages/ci-deployment-and-environments/13.jpg){:\n.center}\n\nPushing code to repository and deploying are separate processes.\n\n{: .note .text-center}\n\n\nImportant detail: The command\n\n[expects\nyou](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#config-settings-and-precedence)\n\nto provide `AWS_ACCESS_KEY_ID` and  `AWS_SECRET_ACCESS_KEY` environment\n\nvariables. Also you might need to specify `AWS_DEFAULT_REGION`.\n\n{: .alert .alert-info}\n\n\nLet’s try to automate it using [GitLab\nCI](/solutions/continuous-integration/).\n\n\n## The first automated deployment\n\n\nWith GitLab, there's no difference on what commands to run.\n\nYou can set up GitLab CI in a way that tailors to your specific needs, as if\nit was your local terminal on your computer. As long as you execute commands\nthere, you can tell CI to do the same for you in GitLab.\n\nPut your script to `.gitlab-ci.yml` and push your code – that’s it: CI\ntriggers\n\na _job_ and your commands are executed.\n\n\nNow, let's add some context to our story: Our website is small, there is\n20-30 daily\n\nvisitors and the code repository has only one default branch: `main`.\n\n\nLet's start by specifying a _job_ with the command from above in the\n`.gitlab-ci.yml` file:\n\n\n```yaml\n\ndeploy:\n  script: aws s3 cp ./ s3://yourbucket/ --recursive --exclude \"*\" --include \"*.html\"\n```\n\n\nNo luck:\n\n![Failed\ncommand](https://about.gitlab.com/images/blogimages/ci-deployment-and-environments/fail1.png){:\n.shadow}\n\n\nIt is our _job_ to ensure that there is an `aws` executable.\n\nTo install `awscli` we need `pip`, which is a tool for Python packages\ninstallation.\n\nLet's specify Docker image with preinstalled Python, which should contain\n`pip` as well:\n\n\n```yaml\n\ndeploy:\n  image: python:latest\n  script:\n  - pip install awscli\n  - aws s3 cp ./ s3://yourbucket/ --recursive --exclude \"*\" --include \"*.html\"\n```\n\n\n![Automated\ndeployment](https://about.gitlab.com/images/blogimages/ci-deployment-and-environments/14.jpg){:\n.center}\n\nYou push your code to GitLab, and it is automatically deployed by CI.\n  {: .note .text-center}\n\nThe installation of `awscli` extends the job execution time, but that is not\na big\n\ndeal for now. If you need to speed up the process, you can always [look for\n\na Docker image](https://hub.docker.com/explore/) with preinstalled `awscli`,\n\nor create an image by yourself.\n\n{: .alert .alert-warning}\n\n\nAlso, let’s not forget about these environment variables, which you've just\ngrabbed\n\nfrom [AWS Console](https://console.aws.amazon.com/):\n\n\n```yaml\n\nvariables:\n  AWS_ACCESS_KEY_ID: \"AKIAIOSFODNN7EXAMPLE\"\n  AWS_SECRET_ACCESS_KEY: \"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\"\ndeploy:\n  image: python:latest\n  script:\n  - pip install awscli\n  - aws s3 cp ./ s3://yourbucket/ --recursive --exclude \"*\" --include \"*.html\"\n```\n\nIt should work, but keeping secret keys open, even in a private repository,\n\nis not a good idea. Let's see how to deal with this situation.\n\n\n### Keeping secret things secret\n\n\nGitLab has a special place for secret variables: **Settings > CI/CD >\nVariables**\n\n\n![Picture of Variables\npage](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674076/Blog/Content%20Images/add-variable-updated.png)\n\n\nWhatever you put there will be turned into **environment variables**.\n\nChecking the \"Mask variable\" checkbox will obfuscate the variable in job\nlogs. Also, checking the \"Protect variable\" checkbox will export the\nvariable to only pipelines running on protected branches and tags. Users\nwith Owner or Maintainer permissions to a project will have access to this\nsection.\n\n\nWe could remove `variables` section from our CI configuration. However,\nlet’s use it for another purpose.\n\n\n### How to specify and use variables that are not secret\n\n\nWhen your configuration gets bigger, it is convenient to keep some of the\n\nparameters as variables at the beginning of your configuration. Especially\nif you\n\nuse them in more than one place. Although it is not the case in our\nsituation yet,\n\nlet's set the S3 bucket name as a\n[**variable**](https://docs.gitlab.com/ee/ci/variables/) for the purpose of\nthis demonstration:\n\n\n```yaml\n\nvariables:\n  S3_BUCKET_NAME: \"yourbucket\"\ndeploy:\n  image: python:latest\n  script:\n  - pip install awscli\n  - aws s3 cp ./ s3://$S3_BUCKET_NAME/ --recursive --exclude \"*\" --include \"*.html\"\n```\n\n\nSo far so good:\n\n\n![Successful\nbuild](https://about.gitlab.com/images/blogimages/ci-deployment-and-environments/build.png){:\n.shadow.medium.center}\n\n\nIn our hypothetical scenario, the audience of your website has grown, so\nyou've hired a developer to help you.\n\nNow you have a team. Let's see how teamwork changes the GitLab CI workflow.\n\n\n## How to use GitLab CI with a team\n\n\nNow, that there are two users working in the same repository, it is no\nlonger convenient\n\nto use the `main` branch for development. You decide to use separate\nbranches\n\nfor both new features and new articles and merge them into `main` when they\nare ready.\n\n\nThe problem is that your current CI config doesn’t care about branches at\nall.\n\nWhenever you push anything to GitLab, it will be deployed to S3.\n\n\nPreventing this problem is straightforward. Just add `only: main` to your\n`deploy` job.\n\n\n![Automated deployment of main\nbranch](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674076/Blog/Content%20Images/15-updated.png){:\n.center}\n\nYou don't want to deploy every branch to the production website but it would\nalso be nice to preview your changes from feature-branches somehow.\n\n{: .note .text-center}\n\n\n### How to set up a separate place for testing code\n\n\nThe person you recently hired, let's call him Patrick, reminds you that\nthere is a featured called\n\n[GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/). It looks like a perfect\ncandidate for\n\na place to preview your work in progress.\n\n\nTo [host websites on GitLab Pages](/blog/gitlab-pages-setup/) your CI\nconfiguration file should satisfy three simple rules:\n\n\n- The _job_ should be named `pages`\n\n- There should be an `artifacts` section with folder `public` in it\n\n- Everything you want to host should be in this `public` folder\n\n\nThe contents of the public folder will be hosted at\n`http://\u003Cusername>.gitlab.io/\u003Cprojectname>/`\n\n{: .alert .alert-info}\n\n\nAfter applying the [example config for plain-html\nwebsites](https://gitlab.com/pages/plain-html/blob/master/.gitlab-ci.yml),\n\nthe full CI configuration looks like this:\n\n\n```yaml\n\nvariables:\n  S3_BUCKET_NAME: \"yourbucket\"\n\ndeploy:\n  image: python:latest\n  script:\n  - pip install awscli\n  - aws s3 cp ./ s3://$S3_BUCKET_NAME/ --recursive --exclude \"*\" --include \"*.html\"\n  only:\n  - main\n\npages:\n  image: alpine:latest\n  script:\n  - mkdir -p ./public\n  - cp ./*.html ./public/\n  artifacts:\n    paths:\n    - public\n  except:\n  - main\n```\n\n\nWe specified two jobs. One job deploys the website for your customers to S3\n(`deploy`).\n\nThe other one (`pages`) deploys the website to GitLab Pages.\n\nWe can name them \"Production environment\" and \"Staging environment\",\nrespectively.\n\n\n![Deployment to two\nplaces](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674076/Blog/Content%20Images/16-updated.png){:\n.center}\n\nAll branches, except main, will be deployed to GitLab Pages.\n\n{: .note .text-center}\n\n\n## Introducing environments\n\n\nGitLab offers\n [support for environments](https://docs.gitlab.com/ee/ci/environments/) (including dynamic environments and static environments),\n and all you need to do it to specify the corresponding environment for each deployment *job*:\n\n```yaml\n\nvariables:\n  S3_BUCKET_NAME: \"yourbucket\"\n\ndeploy to production:\n  environment: production\n  image: python:latest\n  script:\n  - pip install awscli\n  - aws s3 cp ./ s3://$S3_BUCKET_NAME/ --recursive --exclude \"*\" --include \"*.html\"\n  only:\n  - main\n\npages:\n  image: alpine:latest\n  environment: staging\n  script:\n  - mkdir -p ./public\n  - cp ./*.html ./public/\n  artifacts:\n    paths:\n    - public\n  except:\n  - main\n```\n\n\nGitLab keeps track of your deployments, so you always know what is currently\nbeing deployed on your servers:\n\n\n![List of\nenvironments](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674076/Blog/Content%20Images/envs-updated.png){:\n.shadow.center}\n\n\nGitLab provides full history of your deployments for each of your current\nenvironments:\n\n\n![List of deployments to staging\nenvironment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674077/Blog/Content%20Images/staging-env-detail-updated.png){:\n.shadow.center}\n\n\n![Environments](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674077/Blog/Content%20Images/17-updated.png){:\n.center}\n\n\nNow, with everything automated and set up, we’re ready for the new\nchallenges that are just around the corner.\n\n\n## How to troubleshoot deployments\n\n\nIt has just happened again.\n\nYou've pushed your feature-branch to preview it on staging and a minute\nlater Patrick pushed\n\nhis branch, so the staging environment was rewritten with his work. Aargh!!\nIt was the third time today!\n\n\nIdea! \u003Ci class=\"far fa-lightbulb\" style=\"color:#FFD900; font-size:.85em\"\naria-hidden=\"true\">\u003C/i> Let's use Slack to notify us of deployments, so that\npeople will not push their stuff if another one has been just deployed!\n\n\n> Learn how to [integrate GitLab with\nSlack](https://docs.gitlab.com/ee/user/project/integrations/gitlab_slack_application.html).\n\n\n## Teamwork at scale\n\n\nAs the time passed, your website became really popular, and your team has\ngrown from two people to eight people.\n\nPeople develop in parallel, so the situation when people wait for each other\nto\n\npreview something on Staging has become pretty common. \"Deploy every branch\nto staging\" stopped working.\n\n\n![Queue of branches for review on\nStaging](https://about.gitlab.com/images/blogimages/ci-deployment-and-environments/queue.jpg){:\n.center}\n\n\nIt's time to modify the process one more time. You and your team agreed that\nif\n\nsomeone wants to see their changes on the staging\n\nserver, they should first merge the changes to the \"staging\" branch.\n\n\nThe change of `.gitlab-ci.yml` is minimal:\n\n\n```yaml\n\nexcept:\n\n- main\n\n```\n\n\nis now changed to\n\n\n```yaml\n\nonly:\n\n- staging\n\n```\n\n\n![Staging\nbranch](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674077/Blog/Content%20Images/18-updated.png){:\n.center}\n\nPeople have to merge their feature branches before preview on the staging\nserver.\n\n{: .note .text-center}\n\n\nOf course, it requires additional time and effort for merging, but everybody\nagreed that it is better than waiting.\n\n\n### How to handle emergencies\n\n\nYou can't control everything, so sometimes things go wrong. Someone merged\nbranches incorrectly and\n\npushed the result straight to production exactly when your site was on top\nof HackerNews.\n\nThousands of people saw your completely broken layout instead of your shiny\nmain page.\n\n\nLuckily, someone found the **Rollback** button, so the\n\nwebsite was fixed a minute after the problem was discovered.\n\n\n![List of\nenvironments](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674077/Blog/Content%20Images/prod-env-rollback-arrow-updated.png){:\n.shadow.center}\n\nRollback relaunches the previous job with the previous commit\n\n{: .note .text-center}\n\n\nAnyway, you felt that you needed to react to the problem and decided to turn\noff\n\nauto-deployment to Production and switch to manual deployment.\n\nTo do that, you needed to add `when: manual` to your _job_.\n\n\nAs you expected, there will be no automatic deployment to Production after\nthat.\n\nTo deploy manually go to **CI/CD > Pipelines**, and click the button:\n\n\n![Skipped job is available for manual\nlaunch](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674076/Blog/Content%20Images/manual-pipeline-arrow-updated.png){:\n.shadow.center}\n\n\nFast forward in time. Finally, your company has turned into a corporation.\nNow, you have hundreds of people working on the website,\n\nso all the previous compromises no longer work.\n\n\n### Time to start using Review Apps\n\n\nThe next logical step is to boot up a temporary instance of the application\nper feature branch for review.\n\n\nIn our case, we set up another bucket on S3 for that. The only difference is\nthat\n\nwe copy the contents of our website to a \"folder\" with the name of the\n\nthe development branch, so that the URL looks like this:\n\n\n`http://\u003CREVIEW_S3_BUCKET_NAME>.s3-website-us-east-1.amazonaws.com/\u003Cbranchname>/`\n\n\nHere's the replacement for the `pages` _job_ we used before:\n\n\n```yaml\n\nreview apps:\n  variables:\n    S3_BUCKET_NAME: \"reviewbucket\"\n  image: python:latest\n  environment: review\n  script:\n  - pip install awscli\n  - mkdir -p ./$CI_BUILD_REF_NAME\n  - cp ./*.html ./$CI_BUILD_REF_NAME/\n  - aws s3 cp ./ s3://$S3_BUCKET_NAME/ --recursive --exclude \"*\" --include \"*.html\"\n```\n\n\nThe interesting thing is where we got this `$CI_BUILD_REF_NAME` variable\nfrom.\n\nGitLab predefines [many environment\nvariables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\nso that you can use them in your jobs.\n\n\nNote that we defined the `S3_BUCKET_NAME` variable inside the *job*. You can\ndo this to rewrite top-level definitions.\n\n{: .alert .alert-info}\n\n\nVisual representation of this configuration:\n\n![Review apps]![How to use GitLab CI - update - 19 -\nupdated](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674077/Blog/Content%20Images/19-updated.png){:\n.illustration}\n\n\nThe details of the Review Apps implementation varies widely, depending upon\nyour real technology\n\nstack and on your deployment process, which is outside the scope of this\nblog post.\n\n\nIt will not be that straightforward, as it is with our static HTML website.\n\nFor example, you had to make these instances temporary, and booting up these\ninstances\n\nwith all required software and services automatically on the fly is not a\ntrivial task.\n\nHowever, it is doable, especially if you use Docker containers, or at least\nChef or Ansible.\n\n\nWe'll cover deployment with Docker in a future blog post.\n\nI feel a bit guilty for simplifying the deployment process to a simple HTML\nfiles copying, and not\n\nadding some hardcore scenarios. If you need some right now, I recommend you\nread the article [\"Building an Elixir Release into a Docker image using\nGitLab\nCI.\"](/blog/building-an-elixir-release-into-docker-image-using-gitlab-ci-part-1/)\n\n\nFor now, let's talk about one final thing.\n\n\n### Deploying to different platforms\n\n\nIn real life, we are not limited to S3 and GitLab Pages. We host, and\ntherefore,\n\ndeploy our apps and packages to various services.\n\n\nMoreover, at some point, you could decide to move to a new platform and will\nneed to rewrite all your deployment scripts.\n\nYou can use a gem called `dpl` to minimize the damage.\n\n\nIn the examples above we used `awscli` as a tool to deliver code to an\nexample\n\nservice (Amazon S3).\n\nHowever, no matter what tool and what destination system you use, the\nprinciple is the same:\n\nYou run a command with some parameters and somehow pass a secret key for\nauthentication purposes.\n\n\nThe `dpl` deployment tool utilizes this principle and provides a\n\nunified interface for [this list of\nproviders](https://github.com/travis-ci/dpl#supported-providers).\n\n\nHere's how a production deployment _job_ would look if we use `dpl`:\n\n\n```yaml\n\nvariables:\n  S3_BUCKET_NAME: \"yourbucket\"\n\ndeploy to production:\n  environment: production\n  image: ruby:latest\n  script:\n  - gem install dpl\n  - dpl --provider=s3 --bucket=$S3_BUCKET_NAME\n  only:\n  - main\n```\n\n\nIf you deploy to different systems or change destination platform\nfrequently, consider\n\nusing `dpl` to make your deployment scripts look uniform.\n\n\n## Five key takeaways\n\n\n1. Deployment is just a command (or a set of commands) that is regularly\nexecuted. Therefore it can run inside GitLab CI.\n\n2. Most times you'll need to provide some secret key(s) to the command you\nexecute. Store these secret keys in **Settings > CI/CD > Variables**.\n\n3. With GitLab CI, you can flexibly specify which branches to deploy to.\n\n4. If you deploy to multiple environments, GitLab will conserve the history\nof deployments,\n\nwhich allows you to rollback to any previous version.\n\n5. For critical parts of your infrastructure, you can enable manual\ndeployment from GitLab interface, instead of automated deployment.\n\n\n\u003Cstyle>\n\nimg.illustration {\n  padding-left: 12%;\n  padding-right: 12%;\n\n}\n\n@media (max-width: 760px) {\n  img.illustration {\n    padding-left: 0px;\n    padding-right: 0px;\n  }\n}\n\n\u003C/style>\n",[787,788,9],"2024-07-22",{"slug":1390,"featured":6,"template":700},"ci-deployment-and-environments","content:en-us:blog:ci-deployment-and-environments.yml","Ci Deployment And Environments","en-us/blog/ci-deployment-and-environments.yml","en-us/blog/ci-deployment-and-environments",{"_path":1396,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1397,"content":1403,"config":1409,"_id":1411,"_type":13,"title":1412,"_source":15,"_file":1413,"_stem":1414,"_extension":18},"/en-us/blog/combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform",{"title":1398,"description":1399,"ogTitle":1398,"ogDescription":1399,"noIndex":6,"ogImage":1400,"ogUrl":1401,"ogSiteName":685,"ogType":686,"canonicalUrls":1401,"schema":1402},"Combine GitLab webhooks and Twilio for SMS alerts on DevSecOps platform","Configure GitLab webhooks with SMS alerts to instantly get feedback on new and existing issues within a project and enable teams to react quickly to project- and group-level changes.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099013/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2814%29_6VTUA8mUhOZNDaRVNPeKwl_1750099012960.png","https://about.gitlab.com/blog/combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Combine GitLab webhooks and Twilio for SMS alerts on DevSecOps platform\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ted Gieschen\"}],\n        \"datePublished\": \"2024-06-10\",\n      }",{"title":1398,"description":1399,"authors":1404,"heroImage":1400,"date":1406,"body":1407,"category":1119,"tags":1408},[1405],"Ted Gieschen","2024-06-10","We all strive to create the most robust and secure DevSecOps environments\nwhere everyone can collaborate to deliver amazing products for our\ncustomers. But no matter how robust and secure we design our environments we\ncannot exclude the possibility that something might go wrong. When an issue\ndoes occur we want to make sure we can remediate it quickly. To do that it's\nnot only important to document the details of the issue but also get the\nright people notified immediately. In this article, we will set up GitLab\n[webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html)\ntogether with [Twilio's functionality](https://www.twilio.com/en-us) to\n[send SMS alerts](https://www.twilio.com/docs/messaging) to the right\npeople, getting them up to date so they can mitigate problems quickly.\n\n\n## Prerequisites\n\n\n1. A GitLab account: Webhooks aren't restricted by tier, which means this\nfeature can be used with a [Free, Premium or Ultimate\nlicense](https://about.gitlab.com/pricing/) for either [GitLab's SaaS or\nself-managed\noffering](https://docs.gitlab.com/ee/subscriptions/choosing_subscription.html).\nIf you don't have an account yet, you can create one on [our sign-up page](\nhttps://gitlab.com/users/sign_up).\n\n\n2. A Twilio account: To handle the incoming webhook and send an SMS, you\nwill need a Twilio account. If you don't already have one, you can create\none on [Twilio's sign-up page](https://www.twilio.com/try-twilio).\n\n\n3. (Optional) An SMS-capable phone to test the functionality: We will be\ntesting the functionality at the end of this article. If you want to follow\nalong, you will need access to a phone that can receive SMS texts.\n\n\n4. (Optional) A basic understanding of Node.js: We will be handling the\nwebhooks using a serverless function provided by Twilio Functions. This will\nbe written in [Node.js](https://nodejs.org/en/about). Although you can\nsimply copy-paste the functionality, it would be beneficial to understand\nthe basics of Node.js so you can expand functionality in the future.\n\n\n## Building automated SMS notifications\n\n\nNow, let's get hands-on with building real-time SMS notifications.\n\n\nAt a high level, the workflow looks as follows:\n\n\n![SMS\nworkflow](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099023261.png)\n\n\n1. An event is triggered within GitLab. This event is then picked up by\nGitLab's webhook functionality.\n\n2. The information of the event is then sent as a webhook to a [Twilio\nFunction](https://www.twilio.com/docs/serverless/functions-assets/functions).\n\n3. Twilio Functions processes the event data sent by GitLab and creates the\nSMS body with relevant information.\n\n4. When complete, Twilio Functions triggers [Twilio Programmable\nMessaging](https://www.twilio.com/docs/messaging) with the SMS body and\nrecipient information.\n\n5. Twilio Programmable Messaging then sends the SMS with the generated body\nto the recipient.\n\n\n### Set up Twilio SMS\n\n\nWe need to set up our Twilio environment to be able to send SMS. To do this,\nlog in to your Twilio account. If you don't have one just follow the link\nprovided in the prerequisites section above.\n\n\nOnce logged in you will see the Twilio Console, which will look something\nlike this:\n\n\n![Twilio\nconsole](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099023261.png)\n\n\nFrom here, we will head to the left sidebar menu and select __United States\n(US1) > Phone Numbers > Manage > Active numbers__ and then click the \"Buy a\nnumber\" button.\n\n\n![Buy a number\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750099023263.png)\n\n\nYou can select a phone number, which will be the number that notifications\nare sent from. There are some\n[guidelines](https://www.twilio.com/docs/messaging/guides/sending-international-sms-guide)\nspecific to which countries you can send SMS based on the Twilio phone\nnumber you purchase, so please keep that in mind. In this example, I will be\nusing my personal U.S. phone number for this article as the recipient phone\nnumber, so, in this case, I will purchase a U.S. Twilio number. Just make\nsure your phone number has the SMS capability. Once selected, simply click\nthe \"Buy \u003Cphone number>\"  button.\n\n\n![twilio webhooks - image\n4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099023265.png)\n\n\nNext, we just need to make sure Twilio can send SMS to our recipient phone\nnumber by allowing Twilio Programmable Messaging to send SMS to the country\nour recipient phone number is associated with. To do so, head to __[United\nStates (US1) > Messaging > Settings > Geo permissions__ and make sure that\nthe country associated with the recipient's phone number is selected (for\nexample, as I am using my U.S. phone number as the recipient phone number in\nthis blog, I will select United States).\n\n\n![twilio webhooks - image\n5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750099023267.png)\n\n\nClick \"Save geo permissions.\" With that we're all set up to send SMS.\n\n\nNext, let's handle the processing of the webhook and the creation of our SMS\nalerts with Twilio Functions.\n\n\n### Set up Twilio Functions\n\n\nTo process the webhook we will be sending to Twilio, we need to define a\nTwilio Function. To do this, select **United States (US1) > Functions and\nAssets > Functions (Classic) > List** and click \"Create a Function.\" Select\nthe \"Hello SMS\" option in the pop-up and click \"Create.\"\n\n\n![Create a Twilio\nfunction](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099023269.png)\n\n\nNow, let's go ahead and configure our Twilio Function.\n\n\n1. Extend the path for example `/handle-event-webhook`. In my case this\nwould result in the following path:\n`https://daff-mac-7354.twil.io/handle-event-webhook`.\n\n\n2. Disable the option `Check for valid Twilio signature`.\n\n\n3. Adjust the code to the following, making sure to update the values for\n`\u003Cyour personal phone number>` and `\u003Cyour Twilio Phone number>`:\n\n\n``` javascript\n\nexports.handler = function (context, event, callback) {\n  const twilioClient = context.getTwilioClient();\n\n  twilioClient.messages\n    .create({\n      body: `Hi there! There was an update to issue (${event[\"object_attributes\"][\"id\"]}) with title \"${event[\"object_attributes\"][\"title\"]}\" in project ${event[\"repository\"][\"name\"]}. It was just ${event[\"object_attributes\"][\"action\"]}.`,\n      to: \"\u003Cyour personal phone number>\",\n      from: \"\u003Cyour Twilio Phone number>\",\n    })\n    .then((message) => {\n      console.log(\"SMS successfully sent\");\n      console.log(message.sid);\n      return callback(null, `Success! Message SID: ${message.sid}`);\n    })\n    .catch((error) => {\n      console.error(error);\n      return callback(error);\n    });\n};\n\n\n```\n\n\nIt should end up looking like the following:\n\n  ![Configuration for Twilio function](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099023271.jpg)\n\nNow, whenever our endpoint is hit, it should trigger an SMS with a custom\nmessage indicating a change to an existing issue which will represent an\nexample of the various [webhook\nevents](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html)\nwe can configure.\n\n\nNext, let's set our webhooks within GitLab to trigger this endpoint whenever\na change to an issue is made.\n\n\n### Set up GitLab webhooks\n\n\nLog in to your GitLab instance and go to the project you would like to\nconfigure event webhooks in.\n\n\nOnce in the Project, go to **Settings > Webhooks** and click on \"Add new\nwebhook.\"\n\n\n![Screen to add a new\nwebhook](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099023273.png)\n\n\nYou will only need to configure the following fields:\n\n\n1. URL: This should be the endpoint we defined in the previous section. In\nthe previous example that would be\n`https://daff-mac-7354.twil.io/handle-event-webhook`.\n\n\n2. Trigger: In our case, we will be reacting to [issues\nevents](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#issue-events),\nso check \"Issues events.\"\n\n\n![Configuring URL and trigger\nfields](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750099023274.png)\n\n\nWe're all set to test our setup!\n\n\n### Testing\n\n\nWhile in the project that was just configured to react to issues events,\nhead to \"Plan > Issues\" and click on \"New issue.\"\n\n\n![New issue\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750099023276.png)\n\n\nAdd a title and click on \"Create Issue.\"\n\n  ![Create issue screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099023/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750099023278.png)\n\nIf everything is configured correctly, you should get an SMS looking\nsomething like:\n\n\n`Sent from your Twilio trial account - Hi there! There was an update to\nissue (146735617) with title \"GitLab webhook example\" in project Webhooks\nExample. It was just opened.`\n\n\n## Expanding the use case\n\n\nWe've leveraged Twilio's SMS functionality in combination with GitLab\nwebhooks to instantly get feedback on new and existing issues within our\nproject, allowing us to react quickly to any changes that might occur. This\nsimple use case showed how one person could instantly get informed about a\nsingle type of event. However, often we want to inform more people about\nvarious events or be able to react to more than just one type of event (like\nissue creation and updates).\n\n\nThis functionality can be expanded by:\n\n\n1. Sending SMS alerts to multiple people: This can be achieved by extending\nthe Twilio Function to loop through a given array of phone numbers.\n[Twilio's Messaging Service](https://www.twilio.com/docs/messaging/services)\ncan be leveraged to potentially simplify the process of sending SMS to\nvarious phone numbers.\n\n\n2. Handling different event types: Select more types of webhook events in\nthe Project settings to react to other things like\n[comments](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#comment-events),\n[deployments](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#deployment-events),\nor\n[releases](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#release-events).\n\n\n3. Configure on a group level: In this example, we’ve only configured\nwebhooks on a project level. However, if it is relevant to react to events\nacross projects on a group level, this can also be configured, removing the\nneed to change webhook settings for each project.\n\n\n4. Self-host message generation functionality: Leverage [Twilio Server Side\nSDKs](https://www.twilio.com/docs/libraries) instead of Twilio Functions to\nhost the code yourself. This could benefit you if you have restrictions on\nwhere you can host code as well as allow you to more easily connect with the\nrest of your code base likecfetching information from your database to get\nphone numbers for relevant people.\n\n\n> Start [a free trial of GitLab\nUltimate](https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/blog&glm_content=default-saas-trial)\ntoday to test-drive more DevSecOps features.\n",[696,9,694,830,495],{"slug":1410,"featured":90,"template":700},"combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform","content:en-us:blog:combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform.yml","Combine Gitlab Webhooks And Twilio For Sms Alerts On Devsecops Platform","en-us/blog/combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform.yml","en-us/blog/combine-gitlab-webhooks-and-twilio-for-sms-alerts-on-devsecops-platform",{"_path":1416,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1417,"content":1423,"config":1429,"_id":1431,"_type":13,"title":1432,"_source":15,"_file":1433,"_stem":1434,"_extension":18},"/en-us/blog/compose-readers-and-writers-in-golang-applications",{"title":1418,"description":1419,"ogTitle":1418,"ogDescription":1419,"noIndex":6,"ogImage":1420,"ogUrl":1421,"ogSiteName":685,"ogType":686,"canonicalUrls":1421,"schema":1422},"Compose Readers and Writers in Golang applications","GitLab streams terabytes of Git data every hour using Golang abstractions of I/O implementations. Learn how to compose Readers and Writers in Golang apps.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099464/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_639935439_3oqldo5Yt5wPonEJYZOLTM_1750099464124.jpg","https://about.gitlab.com/blog/compose-readers-and-writers-in-golang-applications","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Compose Readers and Writers in Golang applications\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Igor Drozdov\"}],\n        \"datePublished\": \"2024-02-15\",\n      }",{"title":1418,"description":1419,"authors":1424,"heroImage":1420,"date":1426,"body":1427,"category":785,"tags":1428},[1425],"Igor Drozdov","2024-02-15","Every hour, GitLab transfers terabytes of Git data between a server and a client. It is hard or even impossible to handle this amount of traffic unless it is done efficiently in a streaming fashion. Git data is served by Gitaly (Git server), GitLab Shell (Git via SSH), and Workhorse (Git via HTTP(S)). These services are implemented using Go - the language that conveniently provides abstractions to efficiently deal with I/O operations.\n\nGolang's [`io`](https://pkg.go.dev/io) package provides [`Reader`](https://pkg.go.dev/io#Reader) and [`Writer`](https://pkg.go.dev/io#Writer) interfaces to abstract the functionality of I/O implementations into public interfaces.\n\n`Reader` is the interface that wraps the basic `Read` method:\n\n```go\ntype Reader interface {\n\tRead(p []byte) (n int, err error)\n}\n```\n\n`Writer` is the interface that wraps the basic `Write` method.\n\n```go\ntype Writer interface {\n\tWrite(p []byte) (n int, err error)\n}\n```\n\nFor example, [`os`](https://pkg.go.dev/os) package provides an implementation of reading a file. `File` type implements `Reader` and `Writer` interfaces by defining basic [`Read`](https://pkg.go.dev/os#File.Read) and [`Write`](https://pkg.go.dev/os#File.Write) functions.\n\nIn this blog post, you'll learn how to compose Readers and Writers in Golang applications.\n\nFirst, let's read from a file and write its content to [`os.Stdout`](https://cs.opensource.google/go/go/+/master:src/os/file.go;l=66?q=Stdout&ss=go%2Fgo).\n\n```go\nfunc main() {\n\tfile, err := os.Open(\"data.txt\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer file.Close()\n\n\tp := make([]byte, 32 * 1024)\n\tfor {\n\t\tn, err := file.Read(p)\n\n\t\t_, errW := os.Stdout.Write(p[:n])\n\t\tif errW != nil {\n\t\t\tlog.Fatal(errW)\n\t\t}\n\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tlog.Fatal(err)\n\t\t}\n\t}\n}\n```\n\nEach call of the `Read` function fills the buffer `p` with the content from the file, i.e. the file is being consumed in chunks (up to `32KB`) instead of being fully loaded into the memory.\n\nTo simplify this widely used pattern, `io` package conveniently provides [`Copy`](https://pkg.go.dev/io#Copy) function that allows passing content from any `Reader` to any `Writer` and also [handles](https://cs.opensource.google/go/go/+/refs/tags/go1.21.0:src/io/io.go;l=433) additional edge cases.\n\n```go\nfunc main() {\n\tfile, err := os.Open(\"data.txt\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer file.Close()\n\n\tif _, err := io.Copy(os.Stdout, file); err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n```\n\n`Reader` and `Writer` interfaces are used across the whole Golang ecosystem because they facilitate reading and writing content in a streaming fashion. Therefore, gluing together the Readers and Writers with the functions that expect these interfaces as arguments is a frequent problem to solve. Sometimes it's as straightforward as passing content from a Reader into a Writer, but sometimes the content written into a Writer must be represented as a Reader or the content from a reader must be sent into multiple Writers. Let's have a closer look into different use cases and the examples of solving these types of problems in the `GitLab` codebase.\n\n## Reader -> Writer\n\n**Problem**\n\nWe need to pass content from a Reader into a Writer.\n\n![readers and writers - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099494917.png)\n\n**Solution**\n\nThe problem can be solved by using [`io.Copy`](https://pkg.go.dev/io#Copy).\n\n```go\nfunc Copy(dst Writer, src Reader) (written int64, err error)\n```\n\n**Example**\n\n[`InfoRefs*`](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/gitaly/smarthttp.go#L18-35) Gitaly RPCs return a `Reader` and we want to [stream](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/git/info-refs.go#L78-80) its content to a user via HTTP response:\n\n```go\nfunc handleGetInfoRefsWithGitaly(ctx context.Context, responseWriter *HttpResponseWriter, a *api.Response, rpc, gitProtocol, encoding string) error {\n        ...\n        infoRefsResponseReader, err := smarthttp.InfoRefsResponseReader(ctx, &a.Repository, rpc, gitConfigOptions(a), gitProtocol)\n        ...\n        if _, err = io.Copy(w, infoRefsResponseReader); err != nil {\n            return err\n        }\n        ...\n}\n```\n\n## Reader -> Multiple Writers\n\n**Problem**\n\nWe need to pass content from a Reader into multiple Writers.\n\n![readers and writers - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099494917.png)\n\n**Solution**\n\nThe `io` package provides [`io.MultiWriter`](https://pkg.go.dev/io#MultiWriter) function that _converts_ multiple Writers into a single one. When its `Write` function is called, the content is copied to all the Writers ([implementation](https://cs.opensource.google/go/go/+/refs/tags/go1.21.0:src/io/multi.go;l=127)).\n\n```go\nfunc MultiWriter(writers ...Writer) Writer\n```\n\n**Example**\n\nGiven we want to [build](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/upload/destination/multi_hash.go#L13-18) `md5`, `sha1`, `sha256` and `sha512` hashes from the same content. [`Hash`](https://pkg.go.dev/hash#Hash) type is a `Writer`. Using `io.MultiWriter`, we define [`multiHash`](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/upload/destination/multi_hash.go#L43-61) Writer. After the content is [written](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/upload/destination/destination.go#L124-125) to the `multiHash`, we [calculate](https://gitlab.com/gitlab-org/gitlab/blob/57aafb6a886d05c15dd0fa372fb4f008bec014ea/workhorse/internal/upload/destination/multi_hash.go#L63-70) the hashes of all these functions in a single run.\n\nThe simplified version of the example is:\n\n```go\npackage main\n\nimport (\n\t\"crypto/sha1\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n)\n\nfunc main() {\n\ts1 := sha1.New()\n\ts256 := sha256.New()\n\n\tw := io.MultiWriter(s1, s256)\n\tif _, err := w.Write([]byte(\"content\")); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tfmt.Println(s1.Sum(nil))\n\tfmt.Println(s256.Sum(nil))\n}\n```\n\nFor simplicity, we just call `Write` function on a Writer, but when content comes from a Reader, then `io.Copy` can be used as well:\n\n```go\n_, err := io.Copy(io.MultiWriter(s1, s256), reader)\n```\n\n## Multiple Readers -> Reader\n\n**Problem**\n\nWe have multiple Readers and need to sequentially read from them.\n\n![readers and writers - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099494919.png)\n\n**Solution**\n\nThe `io` package provides [`io.MultiReader`](https://pkg.go.dev/io#MultiReader) function that _converts_ multiple Readers into a single one. The Readers are read in the passed order.\n\n```go\nfunc MultiReader(readers ...Reader) Reader\n```\n\nThen this Reader can be used in any function that accepts `Reader` as an argument.\n\n**Example**\n\nWorkhorse [reads](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/cmd/gitlab-resize-image/png/reader.go#L26-38) the first `N` bytes of an image to detect whether it's a PNG file and _puts them back_ by building a Reader from multiple Readers:\n\n```go\nfunc NewReader(r io.Reader) (io.Reader, error) {\n\tmagicBytes, err := readMagic(r)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif string(magicBytes) != pngMagic {\n\t\tdebug(\"Not a PNG - read file unchanged\")\n\t\treturn io.MultiReader(bytes.NewReader(magicBytes), r), nil\n\t}\n\n\treturn io.MultiReader(bytes.NewReader(magicBytes), &Reader{underlying: r}), nil\n}\n```\n\n## Multiple Readers -> Multiple Writers\n\n**Problem**\n\nWe need to pass content from multiple Readers into multiple Writers.\n\n![readers and writers - image 6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099494921.png)\n\n**Solution**\n\nThe solutions above can be generalized on the many-to-many use case.\n\n```go\n_, err := io.Copy(io.MultiWriter(w1, w2, w3), io.MultiReader(r1, r2, r3))\n```\n\n## Reader -> Reader + Writer\n\n**Problem**\n\nWe need to read content from a Reader or pass the Reader to a function and simultaneously write the content into a Writer.\n\n![readers and writers - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099494923.png)\n\n**Solution**\n\nThe `io` package provides [io.TeeReader](https://pkg.go.dev/io#TeeReader) function that accepts a Reader to read from, a Writer to write to, and returns a Reader that can be processed further.\n\n```go\nfunc TeeReader(r Reader, w Writer) Reader\n```\n\nThe [implementation](https://cs.opensource.google/go/go/+/refs/tags/go1.21.4:src/io/io.go;l=610) of the functionality is straightforward. The passed `Reader` and `Writer` are stored in a structure that is a `Reader` itself:\n\n```go\nfunc TeeReader(r Reader, w Writer) Reader {\n\treturn &teeReader{r, w}\n}\n\ntype teeReader struct {\n\tr Reader\n\tw Writer\n}\n```\n\nThe `Read` function implemented for the structure delegates the `Read` to the passed `Reader` and also performs a `Write` to the passed `Writer`:\n\n```\nfunc (t *teeReader) Read(p []byte) (n int, err error) {\n\tn, err = t.r.Read(p)\n\tif n > 0 {\n\t\tif n, err := t.w.Write(p[:n]); err != nil {\n\t\t\treturn n, err\n\t\t}\n\t}\n\treturn\n}\n```\n\n**Example 1**\n\nWe already touched hashing topic in the `Multiple Writers -> Writer` section and `io.TeeReader` is [used](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/upload/destination/destination.go#L124-125) to provide a Writer to create a hash from content. The returned Reader can be further used to upload content to object storage.\n\n**Example 2**\n\nWorkhorse uses `io.TeeReader` to [implement](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/dependencyproxy/dependencyproxy.go#L57-101) Dependency Proxy [functionality](https://docs.gitlab.com/ee/user/packages/dependency_proxy/). Dependency Proxy caches requested upstream images in the object storage. The not-yet-cached use case has the following behavior:\n\n- A user performs an HTTP request.\n- The upstream image is fetched using [`net/http`](https://pkg.go.dev/net/http) and [`http.Response`](https://pkg.go.dev/net/http#Response) provides its content via `Body` field, which is [`io.ReadCloser`](https://pkg.go.dev/io#ReadCloser) (basically an `io.Reader`).\n- We need to send this content back to the user by writing it into [`http.ResponseWriter`](https://pkg.go.dev/net/http#ResponseWriter) (basically an `io.Writer`).\n- We need to simultaniously upload the content to object storage by performing an [`http.Request`](https://pkg.go.dev/net/http#NewRequest) (a function that accepts an `io.Reader`).\n\nAs a result, `io.TeeReader` can be used to glue these primitives together:\n\n```go\nfunc (p *Injector) Inject(w http.ResponseWriter, r *http.Request, sendData string) {\n\t// Fetch upstream data via HTTP\n\tdependencyResponse, err := p.fetchUrl(r.Context(), sendData)\n\t...\n\t// Create a tee reader. Each Read will read from dependencyResponse.Body and simultaneously\n        // perform a Write to w writer\n\tteeReader := io.TeeReader(dependencyResponse.Body, w)\n\t// Pass the tee reader as the body of an HTTP request to upload it to object storage\n\tsaveFileRequest, err := http.NewRequestWithContext(r.Context(), \"POST\", r.URL.String()+\"/upload\", teeReader)\n\t...\n\tnrw := &nullResponseWriter{header: make(http.Header)}\n\tp.uploadHandler.ServeHTTP(nrw, saveFileRequest)\n\t...\n```\n\n## Writer -> Reader\n\n**Problem**\n\nWe have a function that accepts a Writer, and we are interested in the content that the function would write into the Writer. We want to intercept the content and represent it as a Reader to further process it in a streaming fashion.\n\n![readers and writers - image 5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099495/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099494924.png)\n\n**Solution**\n\nThe `io` package provides [`io.Pipe`](https://pkg.go.dev/io#Pipe) function that returns a Reader and a Writer:\n\n```go\nfunc Pipe() (*PipeReader, *PipeWriter)\n```\n\nThe Writer can be used to be passed to the function that accepts a Writer. All the content that has been written into it will be accessible via the reader, i.e. a synchronous in-memory pipe is created that can be used to connect code expecting an `io.Reader` with code expecting an `io.Writer`.\n\n**Example 1**\n\nFor [LSIF](https://lsif.dev/) file [transformation](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/lsif_transformer/parser/parser.go#L68-72) for code navigation we need to:\n\n- [Read](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/lsif_transformer/parser/parser.go#L48-51) content of a zip file.\n- Transform the content and [serialize](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/lsif_transformer/parser/docs.go#L97-112) it into [`zip.Writer`](https://pkg.go.dev/archive/zip#Writer).\n- [Represent](https://gitlab.com/gitlab-org/gitlab/blob/d97ce3baab7fbf459728ce18766fefd3abb8892f/workhorse/internal/lsif_transformer/parser/parser.go#L68-72) the new compressed content as a Reader to be further processed in a streaming fashion.\n\nThe [`zip.NewWriter`](https://pkg.go.dev/archive/zip#NewWriter) function accepts a Writer to which it will write the compressed content. It is handy when we need to pass an open file descriptor to the function to save the content to the file. However, when we need to pass the compressed content via an HTTP request, we need to represent the data as a Reader.\n\n```go\n// The `io.Pipe()` creates a reader and a writer.\npr, pw := io.Pipe()\n\n// The writer is passed to `parser.transform` function which will write\n// the transformed compressed content into it\n// The writing should happen asynchronously in a goroutine because each `Write` to\n// the `PipeWriter` blocks until it has satisfied one or more `Read`s from the `PipeReader`.\ngo parser.transform(pw)\n\n// Everything that has been written into it is now accessible via the reader.\nparser := &Parser{\n\tDocs: docs,\n\tpr:   pr,\n}\n\n// pr is a reader that can be used to read all the data written to the pw writer\nreturn parser, nil\n```\n\n**Example 2**\n\nFor Geo setups [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) proxies all `git push` operations to secondary and redirects them to primary.\n\n- GitLab Shell establishes an SSH connection and defines [`ReadWriter`](https://gitlab.com/gitlab-org/gitlab-shell/blob/7898d8e69daf51a7b6e01052c4516ca70893a2d4/internal/command/readwriter/readwriter.go#L6-7) struct that has `In` field of `io.Reader` type to read data from a user and `Out` field of `io.Writer` type to send response to the user.\n- GitLab Shell performs an HTTP request to `/info/refs` and sends `response.Body` of type `io.Reader` to the user using [`io.Copy`](https://gitlab.com/gitlab-org/gitlab-shell/blob/7898d8e69daf51a7b6e01052c4516ca70893a2d4/internal/command/githttp/push.go#L60)\n- The user reacts to this response by sending data to `In` and GitLab Shell needs to read this data, convert it to a request expected by Git HTTP, and send it as an HTTP request to `/git-receive-pack`. This is where `io.Pipe` becomes useful.\n\n```go\nfunc (c *PushCommand) requestReceivePack(ctx context.Context, client *git.Client) error {\n\t// Define pipeReader and pipeWriter and use pipeWriter to collect all the data\n\t//sent by the user converted to a format expected by Git HTTP.\n\tpipeReader, pipeWriter := io.Pipe()\n\t// The writing happens asynchronously because it's a blocking operation\n\tgo c.readFromStdin(pipeWriter)\n\n\t// pipeReader can be passed as io.Reader and used to read all the data written to pipeWriter\n\tresponse, err := client.ReceivePack(ctx, pipeReader)\n\t...\n\t_, err = io.Copy(c.ReadWriter.Out, response.Body)\n\t...\n}\n\nfunc (c *PushCommand) readFromStdin(pw *io.PipeWriter) {\n\tvar needsPackData bool\n\n\t// Scanner reads the user input line by line\n\tscanner := pktline.NewScanner(c.ReadWriter.In)\n\tfor scanner.Scan() {\n\t\tline := scanner.Bytes()\n\t\t// And writes it to the pipe writer\n\t\tpw.Write(line)\n\t\t...\n\t}\n\n\t// The data that hasn't been processed by a scanner is copied if necessary\n\tif needsPackData {\n\t\tio.Copy(pw, c.ReadWriter.In)\n\t}\n\n\t// Close the pipe writer to signify EOF for the pipe reader\n\tpw.Close()\n}\n```\n\n## Try Golang\n\nGolang provides elegant patterns designed to efficiently process data in a streaming fashion. The patterns can be used to address new challenges or refactor the existing performance issues associated with high memory consumption.\n\n> Learn more about [GitLab and Golang](https://docs.gitlab.com/ee/development/go_guide/).\n",[9,742,1263,744],{"slug":1430,"featured":6,"template":700},"compose-readers-and-writers-in-golang-applications","content:en-us:blog:compose-readers-and-writers-in-golang-applications.yml","Compose Readers And Writers In Golang Applications","en-us/blog/compose-readers-and-writers-in-golang-applications.yml","en-us/blog/compose-readers-and-writers-in-golang-applications",{"_path":1436,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1437,"config":1440,"content":1442,"_id":1449,"_type":13,"title":1450,"_source":15,"_file":1451,"_stem":1452,"_extension":18},"/en-us/blog/comprehensive-guide-to-gitlab-dast",{"title":1438,"description":1439},"A comprehensive guide to GitLab DAST","DevSecOps teams can learn how to implement and configure dynamic application security testing, perform passive/active scans, and set security policies.",{"slug":1441,"featured":90,"template":700},"comprehensive-guide-to-gitlab-dast",{"title":1438,"description":1439,"authors":1443,"heroImage":1445,"date":1446,"category":830,"tags":1447,"body":1448},[1444],"Fernando Diaz","https://res.cloudinary.com/about-gitlab-com/image/upload/v1756122536/akivvcnafog9c4dhhzkp.png","2025-09-17",[830,9,1057],"Modern businesses entirely depend on web-based platforms for customer interactions, financial\ntransactions, data processing, and core business operations. As digital transformation\naccelerates and remote or hybrid work becomes the norm, the attack surface for web applications has\nexpanded dramatically, making them prime targets for cybercriminals. Therefore, securing web applications has become more critical than ever.\n\nWhile static code analysis catches vulnerabilities in source code, it cannot identify\nruntime security issues that emerge when applications interact with real-world\nenvironments, third-party services, and complex user workflows. This is where Dynamic\nApplication Security Testing ([DAST](https://docs.gitlab.com/user/application_security/dast/)) becomes invaluable. GitLab's integrated DAST solution provides teams with automated security testing capabilities directly within their CI/CD pipelines, on a schedule, or on-demand, enabling continuous security validation\nwithout disrupting development workflows.\n\n## Why DAST?\n\nDAST should be implemented because it provides critical runtime security validation by testing applications\nin their actual operating environment, identifying vulnerabilities that static analysis cannot detect.\nAdditionally, GitLab DAST can be seamlessly integrated into shift-left security workflows, and\ncan enhance compliance assurance along with risk management.\n\n### Runtime vulnerability detection\n\nDAST excels at identifying security vulnerabilities that only manifest when applications are running.\nUnlike static analysis tools that examine code at rest, DAST scanners interact with live applications\nas an external attacker would, uncovering issues such as:\n\n- **Authentication and session management flaws** that could allow unauthorized access\n- **Input validation vulnerabilities,** including SQL injection, cross-site scripting (XSS), and command injection\n- **Configuration weaknesses** in web servers, databases, and application frameworks\n- **Business logic flaws** that emerge from complex user interactions\n- **API security issues,** including improper authentication, authorization, and data exposure\n\nDAST complements other security testing approaches to provide comprehensive application security coverage. When combined with Static Application Security Testing ([SAST](https://docs.gitlab.com/user/application_security/sast/)), Software Composition Analysis ([SCA](https://docs.gitlab.com/user/application_security/dependency_scanning/)), manual\npenetration testing, and [many other scanner types](https://about.gitlab.com/solutions/application-security-testing/), DAST fills critical gaps in security validation:\n\n- **Black-box testing perspective** that mimics real-world attack scenarios\n- **Environment-specific testing** that validates security in actual deployment configurations\n- **Third-party component testing,** including APIs, libraries, and external services\n- **Configuration validation** across the entire application stack\n\n### Seamless shift-left security integration\n\nGitLab DAST seamlessly integrates into existing CI/CD pipelines, enabling teams to identify security\nissues early in the development lifecycle. This shift-left approach provides several key benefits:\n\n- **Cost reduction** — Fixing vulnerabilities during development is significantly less expensive than addressing them in production. Studies show that remediation costs can be 10 to 100 times higher in production environments.\n- **Faster time-to-market** — Automated security testing eliminates bottlenecks caused by manual security reviews, allowing teams to maintain rapid deployment schedules while ensuring security standards.\n- **Developer empowerment** — By providing immediate feedback on security issues, DAST helps developers build security awareness and improve their coding practices over time.\n\n### Compliance and risk management\n\nMany regulatory frameworks and industry standards require regular security testing of web applications.\nDAST helps organizations meet compliance requirements for standards such as:\n\n- **PCI DSS** for applications handling payment card data\n- **SOC 2** security controls for service organizations\n- **ISO 27001** information security management requirements\n\nThe automated nature of GitLab DAST ensures consistent, repeatable security testing that auditors can\nrely on, while detailed reporting provides the documentation needed for compliance validation.\n\n## Implementing DAST\n\nBefore implementing GitLab DAST, ensure your environment meets the following requirements:\n\n- **GitLab version and Ultimate subscription** — DAST is available in [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/) and requires GitLab 13.4 or later for full functionality; however, the [latest version](https://about.gitlab.com/releases/categories/releases/) is recommended.\n- **Application accessibility** — Your application must be accessible via HTTP/HTTPS with a publicly reachable URL or accessible within your GitLab Runner's network.\n- **Authentication setup** — If your application requires authentication, prepare test credentials or configure authentication bypass mechanisms for security testing.\n\n### Basic implementation\n\nThe simplest way to add DAST to your pipeline is by including the DAST template in your [`.gitlab-ci.yml`](https://docs.gitlab.com/ci/#step-1-create-a-gitlab-ciyml-file) file\nand providing a website to scan:\n\n```yaml\ninclude:\n  - template: DAST.gitlab-ci.yml\n\nvariables:\n  DAST_WEBSITE: \"https://your-application.example.com\"\n```\n\nThis basic configuration will:\n- Run a DAST scan against your specified website\n- Generate a security report in GitLab's security dashboard\n- Fail the pipeline if high-severity vulnerabilities are detected\n- Store scan results as pipeline artifacts\n\nHowever, it is suggested to gain the full benefit of [CI/CD](https://about.gitlab.com/topics/ci-cd/), you can first deploy the application\nand set DAST to run only after an application has been deployed. The application URL can be\ndynamically created and the DAST job can be configured fully with [GitLab Job syntax](https://docs.gitlab.com/ci/yaml/).\n\n```yaml\nstages:\n  - build\n  - deploy\n  - dast\n\ninclude:\n  - template: Security/DAST.gitlab-ci.yml\n\n# Builds and pushes application to GitLab's built-in container registry\nbuild:\n  stage: build\n  variables:\n    IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA\n  before_script:\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n  script:\n    - docker build -t $IMAGE .\n    - docker push $IMAGE\n\n# Deploys application to your suggested target, setsup the dast site dynamically, requires build to complete\ndeploy:\n  stage: deploy\n  script:\n    - echo \"DAST_WEBSITE=http://your-application.example.com\" >> deploy.env\n    - echo \"Perform deployment here\"\n  environment:\n    name: $DEPLOY_NAME\n    url: http://your-application.example.com\n  artifacts:\n    reports:\n      dotenv: deploy.env\n  dependencies:\n    - build\n\n# Configures DAST to run a an active scan on non-main branches, and a passive scan on the main branches and requires a deployment to complete before it is run\ndast:\n  stage: dast\n  rules:\n    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n      variables:\n        DAST_FULL_SCAN: \"false\"\n    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH\n      variables:\n        DAST_FULL_SCAN: \"true\"\n  dependencies:\n    - deploy\n```\n\nYou can learn from an example by seeing the [Tanuki Shop](https://gitlab.com/gitlab-da/tutorials/security-and-governance/tanuki-shop) demo application, which generates the\nfollowing pipeline:\n\n![Standard DAST Pipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118303/rr3cyxjwyecxbmrdxon6.png)\n\n### Understanding passive vs. active scans\n\nIn the example above we enabled active scanning for non-default branches:\n\n```yaml\n- if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH\n  variables:\n    DAST_FULL_SCAN: \"true\"\n```\n\nGitLab DAST employs two distinct scanning methodologies (passive and active), each serving\ndifferent security testing needs.\n\n**Passive scans** analyze application responses without sending potentially harmful requests. This approach:\n\n- Examines HTTP headers, cookies, and response content for security misconfigurations\n- Identifies information disclosure vulnerabilities like exposed server versions or stack traces\n- Detects missing security headers (CSP, HSTS, X-Frame-options)\n- Analyzes SSL/TLS configuration and certificate issues\n\n**Active scans** send crafted requests designed to trigger vulnerabilities. This approach:\n\n- Tests for injection vulnerabilities (SQL injection, XSS, command injection)\n- Attempts to exploit authentication and authorization flaws\n- Validates input sanitization and output encoding\n- Tests for business logic vulnerabilities\n\n**Note:** The DAST scanner is set to passive by default.\n\nDAST has several configuration options that can be applied via environment variables.\nFor a list of all the possible configuration options for DAST, see the [DAST documentation](https://docs.gitlab.com/user/application_security/dast/browser/configuration/customize_settings/).\n\n### Authentication configuration\n\nDAST requires authentication configuration in CI/CD jobs to achieve complete security coverage. Authentication enables DAST to simulate real attacks and test user-specific features only accessible after login. The DAST job typically authenticates by submitting login forms in a browser, then verifies success before continuing to crawl the application with saved credentials. Failed authentication stops the job.\n\nSupported authentication methods:\n- Single-step login form\n- Multi-step login form\n- Authentication to URLs outside the target scope\n\nHere is an example for a single-step login form in a [Tanuki Shop MR](https://gitlab.com/gitlab-da/tutorials/security-and-governance/tanuki-shop/-/merge_requests/20) which adds\nadmin authentication to non-default branches.\n\n```yaml\ndast:\n  stage: dast\n  before_script:\n    - echo \"DAST_TARGET_URL set to '$DAST_TARGET_URL'\" # Dynamically loaded from deploy job\n    - echo \"DAST_AUTH_URL set to '$DAST_TARGET_URL'\" # Dynamically loaded from deploy jobs\n  rules:\n    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n      variables:\n        DAST_FULL_SCAN: \"false\"\n    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH\n      variables:\n        DAST_FULL_SCAN: \"true\" # run both passive and active checks\n        DAST_AUTH_USERNAME: \"admin@tanuki.local\" # The username to authenticate to in the website\n        DAST_AUTH_PASSWORD: \"admin123\" # The password to authenticate to in the website\n        DAST_AUTH_USERNAME_FIELD: \"css:input[id=email]\" # A selector describing the element used to enter the username on the login form\n        DAST_AUTH_PASSWORD_FIELD: \"css:input[id=password]\" # A selector describing the element used to enter the password on the login form\n        DAST_AUTH_SUBMIT_FIELD: \"css:button[id=loginButton]\" # A selector describing the element clicked on to submit the login form\n        DAST_SCOPE_EXCLUDE_ELEMENTS: \"css:[id=navbarLogoutButton]\" # Comma-separated list of selectors that are ignored when scanning\n        DAST_AUTH_REPORT: \"true\" # generate a report detailing steps taken during the authentication process\n        DAST_REQUEST_COOKIES: \"welcomebanner_status:dismiss,cookieconsent_status:dismiss\" # A cookie name and value to be added to every request\n        DAST_CRAWL_GRAPH: \"true\" # generate an SVG graph of navigation paths visited during crawl phase of the scan\n  dependencies:\n    - deploy-kubernetes\n```\n\nYou can see if the authentication was successful by viewing the job logs:\n\n![Auth logs](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118293/zdxgwb6jmseyzwcjscrz.png)\n\nOnce this job completes it provides an authentication report which includes screenshots of the login page:\n\n![Auth report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118292/idm62deg3ezeehcubmc1.png)\n\nYou can also see more examples on DAST with authentication in our [DAST demos](https://gitlab.com/gitlab-org/security-products/demos/dast/) group.\nTo learn more about how to perform DAST with authentication with your specific requirements, see the [DAST authentication documentation](https://docs.gitlab.com/user/application_security/dast/browser/configuration/authentication/).\n\n## Viewing results in MR\n\nGitLab's DAST seamlessly integrates security scanning into your development workflow\nby displaying results directly within merge requests:\n\n![DAST MR 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118293/rrx4n3pgxi9vmzlas8vp.png)  \n![DAST MR 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118294/rh9vwv6ohoaenpvicujm.png)  \n![DAST MR 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118294/ficelmulsc0r7bijf24m.png)\n\nThese results include comprehensive vulnerability data within MRs to help developers identify and address\nsecurity issues before code is merged. Here's what DAST typically reports:\n\n### Vulnerability details\n* Vulnerability name and type (e.g., SQL injection, XSS, CSRF)\n* Severity level (Critical, High, Medium, Low, Info)\n* CVSS score when applicable\n* Common Weakness Enumeration (CWE) identifier\n* Confidence level of the finding\n\n### Location information\n* URL/endpoint where the vulnerability was detected\n* HTTP method used (GET, POST, etc.)\n* Request/response details showing the vulnerable interaction\n* Parameter names that are vulnerable\n* Evidence demonstrating the vulnerability\n\n#### Technical context\n* Description of the vulnerability and potential impact\n* Proof of concept showing how the vulnerability can be exploited\n* Request/response pairs that triggered the finding\n* Scanner details (which DAST tool detected it)\n\n### Remediation guidance\n* Solution recommendations for fixing the vulnerability\n* References to security standards (OWASP, etc.)\n* Links to documentation for remediation steps\n\n## Viewing results in GitLab Vulnerability Report\n\nFor managing vulnerabilities located in the default (or production) branch, the GitLab Vulnerability Report provides a centralized dashboard for monitoring all security findings (in the default branch) across your entire project or organization. This comprehensive view aggregates all security scan results, offering filtering and sorting capabilities to help security teams prioritize remediation efforts. \n\n![Vulnerability Report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118304/o8jjgngtxqplcgux9h5p.png)\n\nWhen selecting a vulnerability, you are taken to its vulnerability page:\n\n![Vulnerability Page 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118303/rolcgxhe0lh2s54zz2kc.png)  \n![Vulnerability Page 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118303/dubic3yacd5n11ine1vi.png)  \n![Vulnerability Page 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118303/iojrm3zasqxljuybbqcs.png)\n\nJust like in merge requests, the vulnerability page provides comprehensive vulnerability data, as seen above. From here you can triage vulnerabilities by assigning them with a status:\n\n* Needs triage (Default)\n* Confirmed\n* Dismissed (Acceptable risk, False positive, Mitigating control, Used in tests, Not applicable)\n* Resolved\n\nWhen a vulnerability status is changed, the audit log includes a note of who changed it, when it was changed, and the reason it was changed. This comprehensive system allows security teams to efficiently prioritize, track, and manage vulnerabilities throughout their lifecycle with clear accountability and detailed risk context.\n\n## On-demand and scheduled DAST\n\nGitLab provides flexible scanning options beyond standard CI/CD pipeline integration through\non-demand and scheduled DAST scans. On-demand scans allow security teams and developers to\ninitiate DAST testing manually whenever needed, without waiting for code commits or pipeline triggers.\nThis capability is particularly valuable for ad-hoc security assessments, incident response scenarios,\nor when testing specific application features that may not be covered in regular pipeline scans.\n\n![On-demand 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118296/hs3fhn42ceycmd94oaua.png)  \n![On-demand 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118298/wiptmr948xey6rrodosg.png)\n\nOn-demand scans can be configured with custom parameters, target URLs, and scanning profiles, making\nthem ideal for focused security testing of particular application components or newly-deployed features.\nScheduled DAST scans provide automated, time-based security testing that operates independently of\nthe development workflow. These scans can be configured to run daily, weekly, or at custom intervals,\nensuring continuous security monitoring of production applications.\n\n![Scheduling DAST](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118300/dbxgkeahij4fklkpcpck.png)\n\nTo learn how to implement on-demand or scheduled scans within your project, see the\n[DAST on-demand scan documentation](https://docs.gitlab.com/user/application_security/dast/on-demand_scan/)\n\n## DAST in compliance workflows\n\nGitLab's security policies framework allows organizations to enforce consistent security\nstandards across all projects, while maintaining flexibility for different teams and environments.\nSecurity policies enable centralized governance of DAST scanning requirements, ensuring that\ncritical applications receive appropriate security testing without requiring individual project\nconfiguration. By defining security policies at the group or instance level, security teams can\nmandate DAST scans for specific project types, deployment environments, or risk classifications.\n\n**Scan/Pipeline Execution Policies** can be configured to automatically trigger DAST scans based on\nspecific conditions such as merge requests to protected branches, scheduled intervals, or deployment events.\nFor example, a policy might require full active DAST scans for all applications before production deployment,\nwhile allowing passive scans only for development branches. These policies can include custom variables,\nauthentication configurations, and exclusion rules that are automatically applied to all covered projects,\nreducing the burden on development teams and ensuring security compliance.\n\n![Scan Execution Policy](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118299/twe0967sayasvassimf3.png)\n\n**Merge Request Approval Policies** provide an additional layer of security governance by enforcing human\nreview for code changes that may impact security. These policies can be configured to require security team\napproval when DAST scans detect new vulnerabilities, when security findings exceed defined thresholds, or\nwhen changes affect security-critical components. For example, a policy might automatically require approval\nfrom a designated security engineer when DAST findings include high-severity vulnerabilities, while allowing\nlower-risk findings to proceed with standard code review processes.\n\n![MR Approval Policy](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118295/w0odyhf3gnkxis3f61ma.png)\n\nTo learn more about GitLab security policies, see the [policy documentation](https://docs.gitlab.com/user/application_security/policies/).\nAdditionally, for compliance, GitLab provides [Security Inventory](https://docs.gitlab.com/user/application_security/security_inventory/)\nand [Compliance center](https://docs.gitlab.com/user/compliance/compliance_center/), which can allow you to oversee\nif DAST is running in your environment and where it is required.\n\n![Security Inventory](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758118300/hro6gykf7igpnnczmpyg.png)\n\nTo learn more about these features, visit our [software compliance solutions page](https://about.gitlab.com/solutions/software-compliance/).\n\n## Summary\n\nGitLab DAST represents a powerful solution for integrating dynamic security testing into modern development workflows. By implementing DAST in your CI/CD pipeline, your team gains the ability to automatically detect runtime vulnerabilities, maintain compliance with security standards, and build more secure applications without sacrificing development velocity.\n\nThe key to successful DAST implementation lies in starting with basic configuration and gradually expanding to more sophisticated scanning profiles as your security maturity grows. Begin with simple website scanning, then progressively add authentication, custom exclusions, and advanced reporting to match your specific security requirements.\n\nRemember that DAST is most effective when combined with other security testing approaches. Use it alongside static analysis, dependency scanning, and manual security reviews to create a comprehensive security testing strategy. The automated nature of GitLab DAST ensures that security testing becomes a consistent, repeatable part of your development process rather than an afterthought.\n\n> To learn more about GitLab security, check out our [security testing solutions page](https://about.gitlab.com/solutions/application-security-testing/). To get started with GitLab DAST, [sign up for a free trial of GitLab Ultimate today](https://about.gitlab.com/free-trial/devsecops/).\n","content:en-us:blog:comprehensive-guide-to-gitlab-dast.yml","Comprehensive Guide To Gitlab Dast","en-us/blog/comprehensive-guide-to-gitlab-dast.yml","en-us/blog/comprehensive-guide-to-gitlab-dast",{"_path":1454,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1455,"content":1461,"config":1467,"_id":1469,"_type":13,"title":1470,"_source":15,"_file":1471,"_stem":1472,"_extension":18},"/en-us/blog/create-a-workspace-quickly-with-the-gitlab-default-devfile",{"title":1456,"description":1457,"ogTitle":1456,"ogDescription":1457,"noIndex":6,"ogImage":1458,"ogUrl":1459,"ogSiteName":685,"ogType":686,"canonicalUrls":1459,"schema":1460},"Create a workspace quickly with the GitLab default devfile","The GitLab default devfile makes it easier than ever to try out workspaces for new projects. Learn how to share developer environment configurations effortlessly with this tutorial.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097860/Blog/Hero%20Images/Blog/Hero%20Images/REFERENCE%20-%20display%20preview%20for%20blog%20images%20%281%29_2XDPsbkjQ3o6tcdom6IGxI_1750097859914.png","https://about.gitlab.com/blog/create-a-workspace-quickly-with-the-gitlab-default-devfile","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Create a workspace quickly with the GitLab default devfile\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Zhaochen Li\"}],\n        \"datePublished\": \"2025-02-27\",\n      }",{"title":1456,"description":1457,"authors":1462,"heroImage":1458,"date":1464,"body":1465,"category":694,"tags":1466},[1463],"Zhaochen Li","2025-02-27","Software development environments can be complex to set up and maintain. Developers often spend a significant amount of time configuring their local environments with the right dependencies, tools, and settings. GitLab aims to solve this by providing a default devfile that enables you to create workspaces and to start developing quickly.\n\n## GitLab Workspaces\n\nGitLab Workspaces provide isolated development environments for making changes to your GitLab projects without the complexity of setting up local dependencies. Workspaces ensure reproducible development setups, allowing developers to share their environment configurations effortlessly.\n\nBy default, GitLab Workspaces are configured to use the GitLab VS Code fork and include the GitLab Workflow extension. To learn more, visit [the GitLab Workspaces documentation](https://docs.gitlab.com/ee/user/workspace/).\n\n## Understand devfiles\n\nA [**devfile**](https://devfile.io/docs/2.2.0/devfile-ecosystem) is a YAML-based declarative configuration file that defines a project's development environment. It specifies the necessary tools, languages, runtimes, and other components required for development.\n\nPreviously, [setting up a workspace](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/) required a custom devfile at the root of the repository. For example, a `.devfile.yaml` file. A typical devfile looked like this:\n\n![typical default devfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097868/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2025-02-26_at_8.15.58_AM_aHR0cHM6_1750097868229.png)\n\n## GitLab default devfile\n\nStarting in GitLab 17.9, a GitLab default devfile is available for all projects when creating a workspace. This eliminates the need to manually create a devfile before starting a workspace.\nHere is the content of the default devfile:\n\n![GitLab default devfile content](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097868/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2025-02-26_at_8.16.20_AM_aHR0cHM6_1750097868230.png)\n\nWhen creating a workspace with the GitLab UI, the option **Use GitLab default devfile** is always available – regardless of whether custom devfiles exist in the repository. Simply select this option to start exploring GitLab Workspaces with one less setup step.\n\n![Use GitLab default devfile screenshot](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097868/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097868232.png)\n\n## Create your own custom devfiles\nWhile the GitLab default devfile provides a quick way to start a workspace, you may want to customize your development environment to better fit your project's needs. By creating a custom devfile, you can tailor your development environment with the exact tools, dependencies, and configurations needed for your workflow.\n\nConsider creating a custom devfile if you need to:\n\n- Add project-specific dependencies beyond the base development image.\n- Adjust CPU and memory resource limits.\n- Configure multiple containers for additional services like databases.\n- Define custom, project-specific, environment variables.\n- Set up specific port mappings.\n- Integrate specialized development tools like debuggers or language servers.\n\nFor more details, see the [Workspaces devfile documentation](https://docs.gitlab.com/ee/user/workspace/#devfile).\n\n## Read more\n\n- [Build and run containers in Remote Development workspaces](https://about.gitlab.com/blog/build-and-run-containers-in-remote-development-workspaces/)\n- [Use GitLab AI features out-of-the-box in a GitLab Workspace](https://about.gitlab.com/blog/use-gitlab-ai-features-out-of-the-box-in-a-gitlab-workspace/)\n- [Quickstart guide for GitLab Remote Development workspaces](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/)\n- [Enable secure sudo access for GitLab Remote Development workspaces](https://about.gitlab.com/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces/)\n",[765,495,696,9,694],{"slug":1468,"featured":6,"template":700},"create-a-workspace-quickly-with-the-gitlab-default-devfile","content:en-us:blog:create-a-workspace-quickly-with-the-gitlab-default-devfile.yml","Create A Workspace Quickly With The Gitlab Default Devfile","en-us/blog/create-a-workspace-quickly-with-the-gitlab-default-devfile.yml","en-us/blog/create-a-workspace-quickly-with-the-gitlab-default-devfile",{"_path":1474,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1475,"content":1481,"config":1487,"_id":1489,"_type":13,"title":1490,"_source":15,"_file":1491,"_stem":1492,"_extension":18},"/en-us/blog/data-driven-devsecops-exploring-gitlab-insights-dashboards",{"title":1476,"description":1477,"ogTitle":1476,"ogDescription":1477,"noIndex":6,"ogImage":1478,"ogUrl":1479,"ogSiteName":685,"ogType":686,"canonicalUrls":1479,"schema":1480},"Data-driven DevSecOps: Exploring GitLab Insights Dashboards","Learn how to leverage GitLab Insights Dashboards to visualize key metrics, track project progress, and boost team productivity with customizable, data-driven views.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097210/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2811%29_78Dav6FR9EGjhebHWuBVan_1750097210214.png","https://about.gitlab.com/blog/data-driven-devsecops-exploring-gitlab-insights-dashboards","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Data-driven DevSecOps: Exploring GitLab Insights Dashboards\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ricardo Amarilla Villalba\"}],\n        \"datePublished\": \"2024-11-20\",\n      }",{"title":1476,"description":1477,"authors":1482,"heroImage":1478,"date":1484,"body":1485,"category":694,"tags":1486},[1483],"Ricardo Amarilla Villalba","2024-11-20","Metrics and analytics play a crucial role in driving productivity, quality,\nand success. GitLab, as a comprehensive DevSecOps platform, offers powerful\ntools for tracking and visualizing these vital metrics through its Insights\nDashboards. In this article, you'll learn how to use the Insights Dashboards\nin your environment.\n\n\n## Introduction to GitLab metrics and analytics \n\n\nGitLab provides an array of metrics and analytics tools that cover various\naspects of the DevSecOps lifecycle:\n\n\n1. [Productivity\nAnalytics](https://docs.gitlab.com/ee/user/analytics/productivity_analytics.html):\nTrack team velocity, cycle time, and lead time.  \n\n2. [Code Review\nAnalytics](https://docs.gitlab.com/ee/user/analytics/code_review_analytics.html):\nMeasure code quality, test coverage, and review efficiency.  \n\n3. [CI/CD\nAnalytics](https://docs.gitlab.com/ee/user/analytics/ci_cd_analytics.html):\nMonitor pipeline performance and deployment frequency.  \n\n4. [Value Stream\nAnalytics](https://docs.gitlab.com/ee/user/group/value_stream_analytics/):\nVisualize the flow of work from idea to production.  \n\n5. [Insights](https://docs.gitlab.com/ee/user/project/insights/): Explore\nand visualize data about your projects and groups.\n\n\nThese metrics offer invaluable insights into your development process,\nhelping teams identify bottlenecks, optimize workflows, and make data-driven\ndecisions.\n\n\n## Leveraging labels for specific metrics\n\n\nOne of GitLab's most powerful, yet understated features, is Labels, which\nallows you to filter and focus on specific metrics with pinpoint accuracy.\nBy strategically applying labels to issues, merge requests, and epics, you\ncan create custom views that provide targeted insights into your project's\nperformance and progress.\n\n\nLabels in GitLab act as versatile identifiers, allowing you to categorize\nand organize your work items with great flexibility. Whether you're tracking\nfeature development, bug fixes, or team-specific tasks, labels enable you to\nslice and dice your project data in ways that reveal meaningful patterns and\ntrends. This concept parallels the use of tags in cloud deployments, where\nresources are labeled for easier management, cost allocation, and\noperational insights.\n\n\nBy thoughtfully labeling your work items, you're essentially creating a\nsophisticated labeling system that can be leveraged to generate custom\ndashboards and reports. This approach empowers you to zoom in on the metrics\nthat matter most to your team or stakeholders, providing a clear and focused\nview of your project's health and momentum.\n\n\n## How to configure GitLab Insights\n\n\nGitLab Insights allow you to explore and visualize data about your projects\nand groups. They provide valuable analytics on various aspects such as\nissues created and closed during a specified period, average time for merge\nrequests to be merged, and triage hygiene. Insights can be configured for\nboth projects and groups.\n\n\nTo configure Insights:\n\n\n1. For project insights:  \n   * Create a file named `.gitlab/insights.yml` in the root directory of your project.  \n2. For group insights:  \n   * Create a `.gitlab/insights.yml` file in a project that belongs to your group.  \n   * Go to your group's **Settings > General**.  \n   * Expand the **Analytics section** and find the **Insights section**.  \n   * Select the project containing the configuration file and save changes.\n\nThe `.gitlab/insights.yml` file is a YAML file where you define the\nstructure and order of charts in a report, as well as the style of charts to\nbe displayed. Each chart definition includes parameters such as title,\ndescription, type, and query to specify the data source and filtering\nconditions.\n\n\nTo view insights, navigate to **Analyze > Insights** in your project or\ngroup.\n\n\n![View default Insights\nDashboard](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097218/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097217972.png)\n\n\n## Customize merge request insights\n\n\nWhile the default view provides valuable raw information, we can customize\nthe Insights Dashboard to uncover additional layers of information, such as\nwhich team was responsible for each merge request and what type of problem\neach one solved.\n\n\n## Merge request insights for each squad and requirement type\n\n\nMeasuring squad productivity in GitLab can be challenging, especially when\nthe GitLab group and subgroup structure doesn't align perfectly with your\nsquad organization. Here's how to overcome these challenges and effectively\ntrack squad productivity:\n\n\n### **Setting up squad-based metrics**\n\n\n1. **Label creation:** Create unique scope labels for each squad (e.g.,\n`squad::alpha`, `squad::beta`) and each requirement type (e.g., `type::bug`,\n`type::feature`, `type::maintenance`).\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ZUOzORIUJeU?si=T8eHeGizS3blYFHB\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n2. **Label application:** Consistently apply these squad labels to all\nissues and merge requests handled by each squad, regardless of the project\nor group they're in.  \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/fJ9entEBZG8?si=MlM6mKirEdkmwDDJ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n**Hints:**  \n   * Use GitLab API to apply labels massively to existing open, merged, and closed MRs.  \n   * Add/remove/update labels as part of your GitLab CI pipeline.  \n   * Leverage the GitLab Triage Bot to automate the labeling process.  \n\n3. Dashboard setup: Create a `.gitlab/insights.yml` file in your project\nrepository with custom charts for team-specific and type-specific merge\nrequest insights.\n\n\n```\n\n\n## Default Merge Requests insights.yml \n\nmergeRequests:\n  title: Merge requests dashboard\n  charts:\n    - title: Merge requests merged per week \n      type: bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          group_by: week\n          period_limit: 12\n    - title: Merge requests merged per month\n      type: bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          group_by: month\n          period_limit: 3\n\n## Per-teams Merge Requests insights.yml\n\nmergeRequestsTeams:\n  title: Merge requests dashboard per teams\n  charts:\n    - title: Merge requests merged per week \n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          group_by: week\n          period_limit: 12\n          collection_labels:\n            - squad::alpha\n            - squad::beta\n    - title: Merge requests merged per month\n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          group_by: month\n          period_limit: 3\n          collection_labels:\n            - squad::alpha\n            - squad::beta\n\n## Per-teams and Type Merge Requests insights.yml\n\nmergeRequestsTeamsAndType:\n  title: Per Teams and Type - Merge requests dashboard\n  charts:\n    - title: Merge requests merged per week - Squad Alpha\n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          filter_labels: squad::alpha\n          collection_labels:\n            - type::feature\n            - type::bug\n            - type::maintenance\n          group_by: week\n          period_limit: 12\n    - title: Merge requests merged per month - Squad Alpha\n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          filter_labels: squad::alpha\n          collection_labels:\n            - type::feature\n            - type::bug\n            - type::maintenance\n          group_by: month\n          period_limit: 3\n    - title: Merge requests merged per week - Squad Beta\n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          filter_labels: squad::beta\n          collection_labels:\n            - type::feature\n            - type::bug\n            - type::maintenance\n          group_by: week\n          period_limit: 12\n    - title: Merge requests merged per month - Squad Beta\n      type: stacked-bar\n      query:\n        data_source: issuables\n        params:\n          issuable_type: merge_request\n          issuable_state: merged\n          filter_labels: squad::beta\n          collection_labels:\n            - type::feature\n            - type::bug\n            - type::maintenance\n          group_by: month\n          period_limit: 3\n\n```\n\n\nBy implementing these customizations, you can create insightful dashboards\nthat provide a clear view of merge request activity per team and requirement\ntype, allowing you to visualize trends over time, compare performance\nbetween squads, and analyze the distribution of different types of work for\neach squad. \n\n\n![dashboards with view of MR activity per team and requirement\ntype](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097218/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097217972.png)\n\n\n![dashboard comparing performance between\nsquads](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097218/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097217974.png)\n\n\n## Get started today\n\n\nGitLab Insights is just the tip of the iceberg when it comes to metrics and\nanalytics. To explore the full range of GitLab's powerful analytics\nfeatures, including Value Stream Analytics, CI/CD Analytics, and Code Review\nmetrics, check out our Value Stream Management product tour:\n\n\n[![Value Stream Management product\ntour](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097218/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-11-20_at_12.28.08_PM_aHR0cHM6_1750097217976.png)](https://gitlab.navattic.com/vsm)\n\n\n> Ready to start your own metrics journey? Sign up for a [free trial\nof GitLab Ultimate\ntoday](https://gitlab.com/-/trials/new?glm_content=default-saas-trial&glm_source=about.gitlab.com%2F)\nand unlock the full potential of data-driven DevSecOps.\n\n\n## Read more\n\n- [Scheduled Reports Generation tool simplifies value stream\nmanagement](https://about.gitlab.com/blog/new-scheduled-reports-generation-tool-simplifies-value-stream-management/)\n\n- [Getting started with the new GitLab Value Streams\nDashboard](https://about.gitlab.com/blog/getting-started-with-value-streams-dashboard/)\n\n- [AI Impact analytics dashboard measures the ROI of\nAI](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/)\n",[108,495,694,696,9,1016],{"slug":1488,"featured":90,"template":700},"data-driven-devsecops-exploring-gitlab-insights-dashboards","content:en-us:blog:data-driven-devsecops-exploring-gitlab-insights-dashboards.yml","Data Driven Devsecops Exploring Gitlab Insights Dashboards","en-us/blog/data-driven-devsecops-exploring-gitlab-insights-dashboards.yml","en-us/blog/data-driven-devsecops-exploring-gitlab-insights-dashboards",{"_path":1494,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1495,"content":1501,"config":1508,"_id":1510,"_type":13,"title":1511,"_source":15,"_file":1512,"_stem":1513,"_extension":18},"/en-us/blog/demystifying-ci-cd-variables",{"title":1496,"description":1497,"ogTitle":1496,"ogDescription":1497,"noIndex":6,"ogImage":1498,"ogUrl":1499,"ogSiteName":685,"ogType":686,"canonicalUrls":1499,"schema":1500},"GitLab environment variables demystified","CI/CD variables are useful (and flexible) tools to control jobs and pipelines. We unpack everything you need to know about GitLab environment variables.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664679/Blog/Hero%20Images/blog-image-template-1800x945__24_.png","https://about.gitlab.com/blog/demystifying-ci-cd-variables","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab environment variables demystified\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Veethika Mishra\"}],\n        \"datePublished\": \"2021-04-09\",\n      }",{"title":1496,"description":1497,"authors":1502,"heroImage":1498,"date":1504,"body":1505,"category":785,"tags":1506,"updatedDate":1507},[1503],"Veethika Mishra","2021-04-09","There is a lot of flexibility when it comes to defining and using variables\nfor [CI/CD](https://about.gitlab.com/topics/ci-cd/). Variables are extremely\nuseful for controlling jobs and pipelines, and they help you avoid\nhard-coding values in your `.gitlab-ci.yml` configuration file. The\ninformation in this post should weave a larger picture by bringing together\nall (or most) of the information around defining and handling variables,\nmaking it easier to understand the scope and capabilities. Relevant\ndocumentation is linked throughout the post.\n\n\nIn [GitLab CI/CD](https://docs.gitlab.com/ee/ci/), variables can be used to\ncustomize jobs by defining and storing values. When using variables there is\nno need to hard code values. In GitLab, CI/CD variables can be defined by\ngoing to **Settings >> CI/CD >> Variables**, or by simply defining them in\nthe `.gitlab-ci.yml` file.\n\n\nVariables are useful for configuring third-party services for different\ndeployment environments, such as `testing`, `staging`, `production`, etc.\nModify the services attached to those environments by simply changing the\nvariable that points to the API endpoint the services need to use. Also use\nvariables to configure jobs and then make them available as environment\nvariables within the jobs when they run.\n\n\n![GitLab reads the .gitlab-ci.yml file to scan the referenced variable and\nsends the information to the GitLab Runner. The variables are exposed on and\noutput by the\nrunner.](https://about.gitlab.com/images/blogimages/demystifying-ci-cd-variables/variables_processing.jpeg)\n\n\n## The relationship between variables and environments\n\n\nSoftware development as a process includes stages to test a product before\nrolling it out to users.\n[Environments](https://docs.gitlab.com/ee/ci/environments/) are used to\ndefine what those stages look like and it may differ between teams and\norganizations.\n\n\nOn the other hand, variables are data values that are likely to change as a\nresult of user interaction with a product. For example, their age,\npreference, or any input you could possibly think of that might determine\ntheir next step in the product task-flow.\n\n\nWe often hear the term [environment\nvariable](https://docs.gitlab.com/ee/administration/environment_variables.html).\nThese are variables that are defined in a given environment, but outside the\napplication. GitLab CI/CD variables provide developers with the ability to\nconfigure values in their code. Using variables is helpful because it\nensures that the code is flexible. GitLab CI/CD variables allow users to\nmodify an application deployed to a certain environment without making any\nchange to code. It is simple to run tests or even integrate third-party\nservices by changing a configuration environment variable outside the\napplication.\n\n\n## The scope of variables for CI/CD\n\n\n![Order of precedence for CI/CD variables: 1) Manual pipeline run, trigger\nand schedule pipeline variables, 2) Project level, group level, instance\nlevel protected variables, 3) Inherited CI/CD variables, 4) Job level,\nglobal yml defined variables, 5) Deployment variables, 6) Pre-defined CI/CD\nvariables](https://about.gitlab.com/images/blogimages/demystifying-ci-cd-variables/variables_precedence.jpeg)\n\n\n### `.gitlab-ci.yml` defined variables\n\n\nVariables that need to be available in the job environment can be added to\nGitLab. These CI/CD variables are meant to store non-sensitive project\nconfiguration, like the database URL in the `.gitlab-ci.yml` file. Reuse\nthis variable in multiple jobs or scripts, wherever the value is needed. If\nthe value changes, you only need to update the variable once, and the change\nis reflected everywhere the variable is used.\n\n\n### Project CI/CD variables\n\n\nMoving a step above the repository-specific requirements, you can define\nCI/CD variables in [project\nsettings](https://docs.gitlab.com/ee/ci/variables/#for-a-project), which\nmakes them available to CI/CD pipelines. These are stored out of the\nrepository (not in the `.gitlab-ci.yml` file), but are still available to\nuse in the CI/CD configuration and scripts. Storing the variables outside\nthe `.gitlab-ci.yml` file keeps these values limited to a project-only\nscope, and not saved in plain text in the project.\n\n\n### Group and instance CI/CD variables\n\n\nSome variables are relevant at the group level, or even instance level, and\ncould be useful to all projects in a group or instance. Define the variables\nin the [group or instance\nsettings](https://docs.gitlab.com/ee/ci/variables/#for-a-group) so all\nprojects within those scopes can use the variables without actually needing\nto know the value  or having to create the variables for the lower scope.\nFor example, a common value that needs to be updated in multiple projects\ncan be easily managed if it stays up-to-date in a single place.\nAlternatively, multiple projects could use a specific password without\nactually needing to know the value of the password itself.\n\n\n## Jobs and pipelines as environments\n\n\nGitLab CI/CD variables, besides being used as environment variables, also\nwork in the scope of the `.gitlab-ci.yml` configuration file to configure\npipeline behavior, unrelated to any environment. The variables can be stored\nin the project/group/instance settings and be made available to jobs in\npipelines.\n\n\nFor example:\n\n\n```  \n\njob:  \n  rules:  \n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH  \n  script:  \n  - echo \"This job ran on the $CI_COMMIT_BRANCH branch.\"  \n```\n\n\nThe variable `($CI_COMMIT_BRANCH)` in the script section runs in the scope\nof the job in which it was defined. This scope is the \"job environment\" –\nmeaning, when the job starts, the GitLab runner starts up a Docker container\nand runs the job in that environment. The runner will make that variable\n(and all other predefined or custom variables) available to the job, and it\ncan display their value in the log output if needed.\n\n\nBut the variable is **also** used in the `if:` section to determine when the\njob should run. That in itself is not an environment, which is why we call\nthese CI/CD variables. They can be used to dynamically configure your CI/CD\njobs, **as well** as be used as environment variables when the job is\nrunning.\n\n\n## Predefined variables\n\n\nA number of variables are\n[predefined](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\nwhen a GitLab CI/CD pipeline starts. A user can immediately access values\nfor things like commit, project, or pipeline details without needing to\ndefine the variables themselves.\n\n\n## Custom CI/CD variables\n\n\n![Runners can create two kinds of custom CI/CD variables: Type and\nFile.](https://about.gitlab.com/images/blogimages/demystifying-ci-cd-variables/variable_types.jpeg)\n\n\nWhen creating a CI/CD variable in the settings, GitLab gives the user more\nconfiguration options for the variable. Use these extra configuration\noptions for stricter control over more sensitive variables:\n\n\n**Environment scope:** If a variable only ever needs to be used in one\nspecific environment, set it to only ever be available in that environment.\nFor example, you can set a deploy token to only be available in the\n`production` environment.\n\n\n**Protected variables:** Similar to the environment scope, you can set a\nvariable to be available only when the pipeline runs on a protected branch,\nlike your default branch.\n\n\n**Variable type:** A few applications require configuration to be passed to\nit in the form of a file. If a user has an application that requires this\nconfiguration, just set the type of variable as a \"File\". Configuring the\nCI/CD variable this way means that when the runner makes the variable\navailable in the environment, it actually writes it out to a temporary file,\nand stores the path to the file as the value. Next, a user can pass the path\nto the file to any applications that need it.\n\n\nAlong with the listed ways of defining and using variables, GitLab\nintroduced a feature that generates pre-filled variables when there's a need\nto run a pipeline manually. Prefilled variables reduce the chances of\nrunning into an error and makes running the pipeline easier.\n\n\n**Masked variables:** [Masked\nvariables](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable)\nare CI variables that have been **hidden in job logs** to prevent the\nvariable’s value from being displayed. \n\n\n**Masked and hidden variables:** Introduced in [GitLab\n17.4](https://about.gitlab.com/releases/2024/09/19/gitlab-17-4-released/#hide-cicd-variable-values-in-the-ui),\n[Masked and\nhidden](https://docs.gitlab.com/ee/ci/variables/#hide-a-cicd-variable)\nvariables provide the same masking feature from job logs and **keep the\nvalue hidden** **in the Settings UI**. We do not recommend using either of\nthese variables for sensitive data (e.g. secrets) as they can be\ninadvertently exposed. \n\n\n## Secrets\n\n\nA secret is a sensitive credential that should be kept confidential.\nExamples of a secret include:\n\n\n* Passwords  \n\n* SSH keys  \n\n* Access tokens  \n\n* Any other types of credentials where exposure would be harmful to an\norganization\n\n\nGitLab currently enables its users to [use external secrets in\nCI](https://docs.gitlab.com/ee/ci/secrets/), by leveraging HashiCorp Vault,\nGoogle Cloud Secret Manager, and Azure Key Vault to securely manage keys,\ntokens, and other secrets at the project level. This allows users to\nseparate these secrets from other CI/CD variables for security reasons.\n\n\n### GitLab Secrets Manager\n\n\nBesides providing support for external secrets in CI, GitLab is also working\non introducing a [native solution to secrets\nmanagement](https://gitlab.com/groups/gitlab-org/-/epics/10108) to securely\nand conveniently store secrets within GitLab. This solution will also help\ncustomers use the stored secrets in GitLab specific components and\nenvironments, and easily manage access at namespace groups and projects\nlevel. \n\n\n## Read more\n\n* [GitLab native secrets manager to give software supply chain security a\nboost](https://about.gitlab.com/blog/gitlab-native-secrets-manager-to-give-software-supply-chain-security-a-boost/)\n\n\n***Disclaimer:** This blog contains information related to upcoming\nproducts, features, and functionality. It is important to note that the\ninformation in this blog post is for informational purposes only. Please do\nnot rely on this information for purchasing or planning purposes. As with\nall projects, the items mentioned in this blog and linked pages are subject\nto change or delay. The development, release, and timing of any products,\nfeatures, or functionality remain at the sole discretion of GitLab.*\n",[788,696,1263,787,108,9],"2025-01-13",{"slug":1509,"featured":6,"template":700},"demystifying-ci-cd-variables","content:en-us:blog:demystifying-ci-cd-variables.yml","Demystifying Ci Cd Variables","en-us/blog/demystifying-ci-cd-variables.yml","en-us/blog/demystifying-ci-cd-variables",{"_path":1515,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1516,"content":1522,"config":1527,"_id":1529,"_type":13,"title":1530,"_source":15,"_file":1531,"_stem":1532,"_extension":18},"/en-us/blog/deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration",{"title":1517,"description":1518,"ogTitle":1517,"ogDescription":1518,"noIndex":6,"ogImage":1519,"ogUrl":1520,"ogSiteName":685,"ogType":686,"canonicalUrls":1520,"schema":1521},"Deploy a NodeJS Express app with GitLab's Cloud Run integration","This tutorial will show you how to use NodeJS and Express to deploy an application to Google Cloud. This step-by-step guide will have you up and running in less than 10 minutes with the Cloud Run integration.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097892/Blog/Hero%20Images/Blog/Hero%20Images/speedlights_speedlights.png_1750097891963.png","https://about.gitlab.com/blog/deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Deploy a NodeJS Express app with GitLab's Cloud Run integration\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sarah Matthies\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2025-01-13\",\n      }",{"title":1517,"description":1518,"authors":1523,"heroImage":1519,"date":1507,"body":1525,"category":785,"tags":1526},[1524,847],"Sarah Matthies","Are you looking to deploy your NodeJS app to Google Cloud with the least\nmaintenance possible? This tutorial will show you how to utilize GitLab’s\nGoogle Cloud integration to deploy your NodeJS app in less than 10 minutes.\n\n\nTraditionally, deploying an application often requires assistance from\nproduction or DevOps engineers. This integration now empowers developers to\nhandle deployments independently. Whether you’re a solo developer or part of\na large team, this setup gives everyone the ability to deploy their\napplications efficiently.\n\n\n## Overview\n\n\n- Create a new project in GitLab\n\n- Set up your NodeJS application\n\n- Use the Google Cloud integration to create a Service account\n\n- Use the Google Cloud integration to configure Cloud Run via Merge Request\n\n- Enjoy your newly deployed NodeJS app\n\n- Follow the cleanup guide\n\n\n## Prerequisites\n\n- Owner access on a Google Cloud Platform project\n\n- Working knowledge of JavaScript/TypeScript (not playing favorites here!)\n\n- Working knowledge of GitLab CI\n\n- 10 minutes \n\n\n## Step-by-step guide\n\n\n### 1. Create a new project in GitLab\n\n\nWe decided to call our project `nodejs–express-cloud-run` for simplicity.\n\n\n![Create a new\nproject](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097905106.png)\n\n\n### 2. Upload your NodeJS app or use this example to get started.\n\n\n[Demo](https://gitlab.com/demos/templates/nodejs-cloud-run)\n\n\n**Note:** Make sure to include the `cloud-run` [CI\ntemplate](https://gitlab.com/gitlab-org/incubation-engineering/five-minute-production/library/-/raw/main/gcp/cloud-run.gitlab-ci.yml)\nwithin your project.\n\n\n![cloud-run CI template\ninclude](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097905107.png)\n\n\n### 3. Use the Google Cloud integration to create a Service account.\n\n\nNavigate to __Operate > Google Cloud > Create Service account__.\n\n\n![Create Service account\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750097905109.png)\n\n\nAlso configure the region you would like the Cloud Run instance deployed to.\n\n\n![Cloud Run instance deployment region\nselection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097905113.png)\n\n\n### 4. Go to the Deployments tab and use the Google Cloud integration to\nconfigure __Cloud Run via Merge Request__.\n\n\n![Deployments - Configuration of Cloud Run via Merge\nRequest](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097905115.png)\n\n\nThis will open a merge request – immediately merge it.\n\n\n![Merge request for\ndeployment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097905117.png)\n\n\n__Note:__ `GCP_PROJECT_ID`, `GCP_REGION`, `GCP_SERVICE_ACCOUNT`, and\n`GCP_SERVICE_ACCOUNT_KEY` will all be automatically populated from the\nprevious steps.\n\n\n![Variables\nlisting](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097905118.png)\n\n\n### 5. Voila! Check your pipeline and you will see you have successfully\ndeployed to Google Cloud Run using GitLab CI.\n\n\n![Successful deployment to Google Cloud\nRun](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097905119.png)\n\n\nClick the Service URL to view your newly deployed Node server.\n\n\n![View newly deployed Node\nserver](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097905120.png)\n\n\nIn addition, you can navigate to __Operate > Environments__ to see a list of\ndeployments for your environments.\n\n\n![Environments view of deployment\nlist](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image12_aHR0cHM6_1750097905121.png)\n\n\nBy clicking on the environment called `main`, you’ll be able to view a\ncomplete list of deployments specific to that environment.\n\n\n![Main view of deployments to specific\nenvironment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750097905122.png)\n\n\n### 6. Next steps\n\n\nTo get started with developing your Node application, try adding another\nendpoint. For instance, in your `index.js` file, you can add a **/bye**\nendpoint as shown below:\n\n\n```\n\napp.get('/bye', (req, res) => {\n  res.send(`Have a great day! See you!`);\n});\n\n\n```\n\n\nPush the changes to the repo, and watch the `deploy-to-cloud-run` job deploy\nthe updates. Once it’s complete, go back to the Service URL and navigate to\nthe **/bye** endpoint to see the new functionality in action.\n\n\n![Bye\nmessage](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097905/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750097905123.png)\n\n\n## Follow the cleanup guide\n\n\nTo prevent incurring charges on your Google Cloud account for the resources\nused in this tutorial, you can either delete the specific resources or\ndelete the entire Google Cloud project. For detailed instructions, refer to\nthe [cleanup guide\nhere](https://docs.gitlab.com/ee/tutorials/create_and_deploy_web_service_with_google_cloud_run_component/#clean-up).\n\n\n> Read more of these helpful [tutorials from GitLab solutions\narchitects](https://about.gitlab.com/blog/tags/solutions-architecture/).\n",[108,1264,232,1016,9],{"slug":1528,"featured":90,"template":700},"deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration","content:en-us:blog:deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration.yml","Deploy A Nodejs Express App With Gitlabs Cloud Run Integration","en-us/blog/deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration.yml","en-us/blog/deploy-a-nodejs-express-app-with-gitlabs-cloud-run-integration",{"_path":1534,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1535,"content":1541,"config":1547,"_id":1549,"_type":13,"title":1550,"_source":15,"_file":1551,"_stem":1552,"_extension":18},"/en-us/blog/deploy-a-server-using-go-with-gitlab-google-cloud",{"title":1536,"description":1537,"ogTitle":1536,"ogDescription":1537,"noIndex":6,"ogImage":1538,"ogUrl":1539,"ogSiteName":685,"ogType":686,"canonicalUrls":1539,"schema":1540},"Deploy a server using Go with GitLab + Google Cloud","This tutorial shows how to use GitLab’s Google Cloud integration to deploy a Golang server in less than 10 minutes, helping developers become more independent and efficient.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098028/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945_fJKX41PJHKCfSOWw4xQxm_1750098028126.png","https://about.gitlab.com/blog/deploy-a-server-using-go-with-gitlab-google-cloud","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Deploy a server using Go with GitLab + Google Cloud\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Claire Champernowne\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2025-01-28\",\n      }",{"title":1536,"description":1537,"authors":1542,"heroImage":1538,"date":1544,"body":1545,"category":694,"tags":1546},[1543,847],"Claire Champernowne","2025-01-28","Deploying an application to the cloud often requires assistance from production or DevOps engineers. GitLab's Google Cloud integration empowers developers to handle deployments independently. In this tutorial, you'll learn how to deploy a server to Google Cloud in less than 10 minutes using Go. Whether you’re a solo developer or part of a large team, this setup allows you to deploy applications efficiently.\n\n## You'll learn how to:\n\n1. Create a new project in GitLab\n2. Create a Go server utilizing `main.go`\n3. Use the Google Cloud integration to create a Service account\n4. Use the Google Cloud integration to create Cloud Run via a merge request\n5. Access your newly deployed Go server\n6. Clean up your environment\n\n## Prerequisites\n\n- Owner access on a Google Cloud Platform project\n- Working knowledge of Golang\n- Working knowledge of GitLab CI\n- 10 minutes\n\n## Step-by-step Golang server deployment to Google Cloud\n\n### 1. Create a new blank project in GitLab.\n\nWe decided to call our project `golang-cloud-run` for simplicity.\n\n![Create a new blank project in GitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098035249.png)\n\n### 2. Create a server utilizing this `main.go` demo.\n\nFind the `main.go` demo [here](https://gitlab.com/demos/applications/golang-cloud-run).\n\n```\n// Sample run-helloworld is a minimal Cloud Run service.\npackage main\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\tlog.Print(\"starting server...\")\n\thttp.HandleFunc(\"/\", handler)\n\n\t// Determine port for HTTP service.\n\tport := os.Getenv(\"PORT\")\n\tif port == \"\" {\n\t\tport = \"8080\"\n\t\tlog.Printf(\"defaulting to port %s\", port)\n\t}\n\n\t// Start HTTP server.\n\tlog.Printf(\"listening on port %s\", port)\n\tif err := http.ListenAndServe(\":\"+port, nil); err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\nfunc handler(w http.ResponseWriter, r *http.Request) {\n\tname := os.Getenv(\"NAME\")\n\tif name == \"\" {\n\t\tname = \"World\"\n\t}\n\tfmt.Fprintf(w, \"Hello %s!\\n\", name)\n}\n```\n\n### 3. Use the Google Cloud integration to create a Service account.\n\nNavigate to **Operate \\> Google Cloud \\> Create Service account**.\n\n![Golang tutorial - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098036/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750098035250.png)\n\n### 4. Configure the region you would like the Cloud Run instance deployed to.\n\n![Golang tutorial - image10](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098035252.png)\n\n### 5. Use the Google Cloud integration to configure Cloud Run via Merge Request.\n\n![Golang tutorial - image4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098035254.png)\n\n### 6. This will open a merge request. Immediately merge the MR.\n\n![Golang tutorial - image6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098036/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098035257.png)\n\nThis merge request adds a CI/CD deployment job to your pipeline definition. In our case, this is also creating a pipeline definition, as we didn’t have one before.\n\n**Note:** The CI/CD variables `GCP_PROJECT_ID`, `GCP_REGION`, `GCP_SERVICE_ACCOUNT`, `GCP_SERVICE_ACCOUNT_KEY` will all be automatically populated from the previous steps. \n\n![Golang tutorial - image7](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098035259.png)\n\n### 7. Voila! Check your pipeline and you will see you have successfully deployed to Google Cloud Run utilizing GitLab CI.\n\n![Golang tutorial - image2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098035261.png)\n\n\u003Cbr>\n\n![Golang tutorial - image3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098035262.png)\n\n## 8. Click the Service URL to view your newly deployed server.\n\nAlternatively, you can navigate to **Operate \\> Environments** to see a list of deployments for your environments.\n\n![Golang tutorial - image5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098035264.png)\n\nBy clicking on the environment called **main**, you’ll be able to view a complete list of deployments specific to that environment.\n\n![Golang tutorial - image8](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098035/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098035265.png)\n\n## Next steps\n\nTo get started with developing your Go application, try adding another endpoint. For instance, in your `main.go` file, you can add a `/bye` endpoint as shown below (don’t forget to register the new handler function in main!):\n\n```\nfunc main() {\n\tlog.Print(\"starting server...\")\n\n\thttp.HandleFunc(\"/\", handler)\n\thttp.HandleFunc(\"/bye\", byeHandler)\n```\n\n```\nfunc byeHandler(w http.ResponseWriter, r *http.Request) {\n\tname := os.Getenv(\"NAME\")\n\tif name == \"\" {\n\t\tname = \"World\"\n\t}\n\tfmt.Fprintf(w, \"Bye %s!\\n\", name)\n}\n```\n\nYour `main.go` file should now look something like this:\n\n```\n// Sample run-helloworld is a minimal Cloud Run service.\npackage main\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\tlog.Print(\"starting server...\")\n\n\thttp.HandleFunc(\"/\", handler)\n\n\thttp.HandleFunc(\"/bye\", byeHandler)\n\n\t// Determine port for HTTP service.\n\tport := os.Getenv(\"PORT\")\n\tif port == \"\" {\n\t\tport = \"8080\"\n\t\tlog.Printf(\"defaulting to port %s\", port)\n\t}\n\n\t// Start HTTP server.\n\tlog.Printf(\"listening on port %s\", port)\n\tif err := http.ListenAndServe(\":\"+port, nil); err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\nfunc handler(w http.ResponseWriter, r *http.Request) {\n\tname := os.Getenv(\"NAME\")\n\tif name == \"\" {\n\t\tname = \"World\"\n\t}\n\tfmt.Fprintf(w, \"Hello %s!\\n\", name)\n}\n\nfunc byeHandler(w http.ResponseWriter, r *http.Request) {\n\tname := os.Getenv(\"NAME\")\n\tif name == \"\" {\n\t\tname = \"World\"\n\t}\n\tfmt.Fprintf(w, \"Bye %s!\\n\", name)\n}\n```\n\nPush the changes to the repo, and watch the `deploy-to-cloud-run job` deploy the updates. Once it’s complete, go back to the Service URL and navigate to the `/bye` endpoint to see the new functionality in action.\n\n## Clean up the environment\n\nTo prevent incurring charges on your Google Cloud account for the resources used in this tutorial, you can either delete the specific resources or delete the entire Google Cloud project. For detailed instructions, refer to the [cleanup guide](https://docs.gitlab.com/ee/tutorials/create_and_deploy_web_service_with_google_cloud_run_component/#clean-up).\n\n> Discover more tutorials like this in our [Solutions Architecture](https://about.gitlab.com/blog/tags/solutions-architecture/) area.\n",[851,495,9,1016,694,696],{"slug":1548,"featured":6,"template":700},"deploy-a-server-using-go-with-gitlab-google-cloud","content:en-us:blog:deploy-a-server-using-go-with-gitlab-google-cloud.yml","Deploy A Server Using Go With Gitlab Google Cloud","en-us/blog/deploy-a-server-using-go-with-gitlab-google-cloud.yml","en-us/blog/deploy-a-server-using-go-with-gitlab-google-cloud",{"_path":1554,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1555,"content":1561,"config":1567,"_id":1569,"_type":13,"title":1570,"_source":15,"_file":1571,"_stem":1572,"_extension":18},"/en-us/blog/deploy-remix-with-gitlab-and-cloudflare",{"title":1556,"description":1557,"ogTitle":1556,"ogDescription":1557,"noIndex":6,"ogImage":1558,"ogUrl":1559,"ogSiteName":685,"ogType":686,"canonicalUrls":1559,"schema":1560},"How to publish a Remix app to the edge with GitLab and Cloudflare","Learn how to deploy a Remix app with GitLab and Cloudflare Workers.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682517/Blog/Hero%20Images/ryoji-hayasaka-0UZj73PQVew-unsplash.jpg","https://about.gitlab.com/blog/deploy-remix-with-gitlab-and-cloudflare","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to publish a Remix app to the edge with GitLab and Cloudflare\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Janis Altherr\"}],\n        \"datePublished\": \"2022-11-21\",\n      }",{"title":1556,"description":1557,"authors":1562,"heroImage":1558,"date":1564,"body":1565,"category":785,"tags":1566},[1563],"Janis Altherr","2022-11-21","[Remix](https://remix.run) has had a significant impact in the frontend\nspace. \n\nAfter eons of backend frameworks providing some sort of frontend options\nthat are more or \n\nless hated by frontend engineers, followed by frontend frameworks that \n\nrequired a separate API for the most simple tasks that were a pain to \n\nmaintain, now suddenly there are frontend frameworks that you can write\n\nbackend code with.\n\n\nThis is ideal as an application scales: Remix offers the comfort of writing \n\nserver-side code, but should the business logic start to exceed the \n\ncapabilities of Remix, it's easy to move code to an API on a per-request\nbasis. \n\nThis comes without the need to rewrite the entire application logic, while \n\nstill retaining server-side-rendering or even pre-rendering capabilities!\n\n\nThe most performant way to deploy a Remix app is to the edge. This means \n\nthat small instances of your Remix app are run on a server close to the\nrequesting\n\nuser. An edge network consists of hundreds of \n\nservers all over the world, so you can be sure the network latency for the \n\nuser stays low.\n\n\nCurrently the most popular edge service capable of running Remix apps are \n\nCloudflare Workers. Not only does Cloudflare offer a generous free tier, \n\nWorkers are also extremely easy to deploy using GitLab CI/CD. \n\nHere's how to create a Remix app and then deploy it to Cloudflare Workers.\n\n\n## Create your Remix app\n\n\nCreate your Remix app locally using:\n\n\n```bash\n\nnpx create-remix@latest \u003Cmy-app-name>\n\n```\n\n\nThe CLI will now guide you through a series of questions. Some of those you \n\nmay answer as you prefer, but answer the following questions as indicated \n\nbelow:\n\n\n```text\n\n? What type of app do you want to create? \n\n> choose \"Just the Basics\"\n\n\n? Where do you want to deploy? [...]\n\n> choose \"Cloudflare Workers\"\n\n\n? Do you want me to run `npm install`?\n\n> answer \"Yes\"\n\n```\n\n\nInitialize the repository and add the first commit:\n\n\n```shell\n\ngit init\n\ngit add .\n\ngit commit -m \"initial commit\"\n\n```\n\n\n## Create the project in GitLab\n\n\nYou can't push the code as we have yet to set up the remote repository.\n\nVisit GitLab and create a new project. When asked, select \"Create blank \n\nproject.\"\n\n\nIn the project setup dialog, select `Edge Computing` as the `Deployment \n\ntarget`. Choose the visibility level however you like as this affects your \n\nsource code visibility.\n\n\nMake sure you unset the checkbox next to **Initialize repository with a \n\nREADME**, otherwise GitLab will begin a new Git history that you will have\nto reconcile\n\nwith your existing local one.\n\n\nOnce the project is set up, follow the instructions on how to add an \n\nexisting repository – if you've followed the above instructions to the\nletter \n\nyou don't have an existing remote yet, so you can run this simplified set\nof \n\ncommands:\n\n\n```shell\n\ngit remote add origin \u003Cgit-project-url>\n\ngit push -u origin main\n\n```\n\n\n## Configure Cloudflare\n\n\nNow set up your Cloudflare account to enable deployments from GitLab. \n\n[Login](https://dash.cloudflare.com/login) or [create an\naccount](https://dash.cloudflare.com/sign-up).\n\n\n### Subscribe to a Workers plan\n\n\nIf you are creating a Worker for the first time, you will have to sign up\nfor a Workers plan in Cloudflare.\n\n\nIn the Cloudflare dashboard's left sidebar click the entry **Workers**. Let \n\nCloudflare guide you through the setup.\n\n\n![Screenshot: Signing up for Workers in\nCloudflare](https://about.gitlab.com/images/blogimages/remix-cloudflare/workers_onboarding.png)\n\n\nOnce you're back to the Workers overview page, continue below.\n\n\n### Obtain an API token\n\n\nTo be able to deploy your Cloudflare Worker from a GitLab pipeline you will\nneed\n\nan API token. To do so, log in to the Cloudflare dashboard, then open the\n[API \n\ntokens page](https://dash.cloudflare.com/profile/api-tokens) (or find it \n\nmanually via the **user icon** > **My Profile** > **Api Tokens**).\n\n\nClick **Create Token**. Find **Edit Cloudflare Workers**, click **use \n\ntemplate**.\n\n\n![Screenshot: Select API Token template \"Edit Cloudflare\nWorkers\"](https://about.gitlab.com/images/blogimages/remix-cloudflare/api_token_template_selection.png)\n\n\nUnder **Account Resources** choose *Include* and your account name.\n\n\nUnder **Zone Resources** choose *Include*, *Specific Zone* and your site's \n\ndomain. If you haven't set up a domain, you can use a less specific rule \n\nsuch as *All zones from an account*, although we don't recommend doing this;\nthe API token could potentially be used beyond its scope if you add more\nzones to your Cloudflare account later.\n\n\n![Screenshot: API Token Account and Zone\nSettings](https://about.gitlab.com/images/blogimages/remix-cloudflare/api_token_rules.png)\n\n\n**Note:** If you have more than one account associated with the API token\nused \n\nduring deployment, you will have to update your project's `wrangler.toml`\nfile\n\nto use the correct account. [Read more in the Cloudflare\ndocumentation](https://developers.cloudflare.com/workers/wrangler/ci-cd/#account-id).\n\n\nOnce you're done setting up the API token, click **Continue to summary**, \n\nand verify your selections. It should look like this:\n\n\n![Screenshot: API Token Summary\nView](https://about.gitlab.com/images/blogimages/remix-cloudflare/api_token_summary.png)\n\n\nIf you're happy, click **Create Token**. Cloudflare will then show you the\nnew \n\ntoken. \n\n\nCopy the token and save it in GitLab: Open your project in GitLab, then \n\nvisit **Settings** > **CI/CD**. Find **Variables** and click **Expand**.\nClick \n\n**Add Variable**.\n\n\nIn the **Key** field, enter `CLOUDFLARE_API_TOKEN`.\n\nIn the **Value** field, paste the API token from Cloudflare.\n\n\nNow make sure your token isn't leaked in any logs: Check both **Protect** \n\nand **Mask**. When done, click **Add Variable**.\n\n\n![Adding a Variable in\nGitLab](https://about.gitlab.com/images/blogimages/remix-cloudflare/adding_cf_api_token_as_variable.gif)\n\n\n## Create the deployment pipeline\n\n\nThe last step is to create a GitLab pipeline. In your local repository root \n\nfolder, create a file named `.gitlab-ci.yml` and add the following content:\n\n\n```yaml\n\nstages:\n\n- deploy\n\n\ndeploy-worker:\n    image: node:lts\n    stage: deploy\n    environment: production\n    before_script:\n      # install dependencies\n      - npm ci\n    script:\n      - npm run deploy\n    rules:\n      # This rule triggers this job after any push to the default branch\n      - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n```\n\n\nTo learn more about how to configure your GitLab pipeline, read our \n\n[documentation](https://docs.gitlab.com/ee/ci/).\n\n\nNow add the file to the repository:\n\n\n```shell\n\ngit add .gitlab-ci.yml\n\ngit commit -m \"Add Deployment Pipeline\"\n\ngit push\n\n```\n\n\nThis last push will immediately run this pipeline. To monitor the pipeline \n\nprogress, open GitLab. In the left sidebar, find **CI/CD** > **Pipelines**. \n\nOnce the pipeline is marked as _passed_, your Remix site is live!\n\n\nIf you've used the create-app instructions from this blogpost, your app\nshould \n\nhave been configured to use the app name as the Worker's name. Check the \n\n`name` setting in your project's `wrangler.toml`.\n\n\nGo to `https://\u003Cworker-name>.\u003Ccloudflare-account-name>.workers.dev` to see \n\nyour Remix site in action. Congratulations!\n\n\nIn your Cloudflare dashboard, you can monitor your new app by selecting \n\n**Workers** from the left sidebar and then clicking on the Worker with the \n\nname of your app.\n\n\nFrom now on, any push to your repositories default branch will\nautomatically \n\nbe built and deployed to Cloudflare. \n\n\n### Use a custom Domain for your app\n\n\nIf you want to use your own domain, set up your website as a resource now.\n\n\nIn the left sidebar, click on **Websites**. In the main window, find and\nclick\n\nthe **Add Site** button.\n\n\n![Screenshot: Add a new site in\nCloudflare](https://about.gitlab.com/images/blogimages/remix-cloudflare/add_site.png)\n\n\nNow enter your site's domain. Select a plan that suits your needs.\n\nFollow the DNS setup instructions provided on the following pages.\n\n\nOnce you have set up your domain as a website in Cloudflare, go to the \n\nwebsite settings. (In the left sidebar click **Websites**, then select your \n\nsite).\n\n\n![Screenshot: Find your website on the Cloudflare\nDashboard](https://about.gitlab.com/images/blogimages/remix-cloudflare/add_route_step_1.png)\n\n\nThe left sidebar now shows the detail navigation for the selected website. \n\nClick **Workers Routes**, then click **Add Route**. \n\n\n![Screenshot: Add a new route to your\nSite](https://about.gitlab.com/images/blogimages/remix-cloudflare/add_route_step_2.png)\n\n\nIn the Add Route Modal you can add a dynamic pattern to let Cloudflare know\nwhich requests to route to your Worker. \n\nFor Remix apps that's usually all of them, so if your site's domain is \n\n`my-site.com`, use `my-site.com/*`. You can also redirect all subdomain \n\nrequests to the worker by using `*.my-site.com/*` (this is useful if you \n\nwould like to also serve your site at `www.my-site.com`).\n\n\nUnder **Service**, select your newly created Worker.\n\nUnder **Environment**, select **production**.\n\n\nClick \"Save\".\n\n\n![Screenshot: Add route\nmodal](https://about.gitlab.com/images/blogimages/remix-cloudflare/add_route_step_3.png)\n\n\nOnce the DNS servers are updated, your Remix site should be accessible with \n\nyour custom domain.\n\n\n\n## Read More\n\n\n- [Learn more about Cloudflare\nWorkers](https://developers.cloudflare.com/workers/wrangler/configuration/)\n\n- [Check out the Remix docs](https://remix.run/docs/en/v1)\n\n- [Learn about GitLab pipelines](https://docs.gitlab.com/ee/ci/)\n",[9,232,694],{"slug":1568,"featured":6,"template":700},"deploy-remix-with-gitlab-and-cloudflare","content:en-us:blog:deploy-remix-with-gitlab-and-cloudflare.yml","Deploy Remix With Gitlab And Cloudflare","en-us/blog/deploy-remix-with-gitlab-and-cloudflare.yml","en-us/blog/deploy-remix-with-gitlab-and-cloudflare",{"_path":1574,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1575,"content":1581,"config":1587,"_id":1589,"_type":13,"title":1590,"_source":15,"_file":1591,"_stem":1592,"_extension":18},"/en-us/blog/deploy-shopify-themes-with-gitlab",{"title":1576,"description":1577,"ogTitle":1576,"ogDescription":1577,"noIndex":6,"ogImage":1578,"ogUrl":1579,"ogSiteName":685,"ogType":686,"canonicalUrls":1579,"schema":1580},"How to deploy Shopify themes with GitLab","Streamline your development workflow by configuring auto deployments for Shopify themes with GitLab pipelines.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683288/Blog/Hero%20Images/storefront.jpg","https://about.gitlab.com/blog/deploy-shopify-themes-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to deploy Shopify themes with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Austin Regnery\"}],\n        \"datePublished\": \"2022-04-14\",\n      }",{"title":1576,"description":1577,"authors":1582,"heroImage":1578,"date":1584,"body":1585,"category":785,"tags":1586},[1583],"Austin Regnery","2022-04-14","[1.75 million sellers are using Shopify's eCommerce\nplatform](https://backlinko.com/shopify-stores), and every one of these\nonline stores has a codebase that lives somewhere. You may have encountered\nsome challenges while scaling your development efforts at your organization\nwhile working within Shopify. Setting up a process for repeatable\ndeployments with GitLab can keep everything streamlined and safe. No one\nwants something going live in production before it's ready.\n\n\nHere's a simple development flow you are going to be able to replicate using\nGitLab CI/CD pipelines for Shopify theme deployments.\n\n\n1. Develop locally on a feature branch until you are happy with your local\nchanges\n\n2. Merge your `feature` branch into your `main` branch → This will update\nthe staging theme in Shopify\n\n3. When everything is ready to go live, create a new tag and push it to\nGitLab → The live theme will be updated automatically 🎉\n\n\nThis tutorial assumes you have set up a repository in a GitLab project.\n\n\n## 1. Add your variables\n\n\nFor security purposes, you don't want to store your credentials for your\nShopify site in your configuration file. You can use variables in GitLab to\nhandle that.\n\n\nUse the [ThemeKit CLI](https://shopify.github.io/themekit/configuration/) to\nretrieve all the available theme IDs from your Shopify store by entering\nthis into your command line:\n\n\n```curl\n\ntheme get --list -p=[shopify-api-access-token] -s=[your-store.myshopify.com]\n\n```\n\n\n> **Help:** [Generate API credentials in\nShopify](https://shopify.dev/apps/auth/basic-http#step-2-generate-api-credentials)\n\n\nOpen your project in GitLab, navigate to `/settings/ci_cd`, and open the\nvariables section.\n\n\nAdd four unique variables with their corresponding keys and values\n\n\n| Key | Value |\n\n| --- | ----- |\n\n| `STAGING_THEME_ID`     | [staging-theme-id-number]    |\n\n| `PRODUCTION_THEME_ID`  | [production-theme-id-number] |\n\n| `SHOP_WEB_ADDRESS`     | [your-store.myshopify.com]   |\n\n| `SHOPIFY_API_ACCESS_TOKEN` | [shopify-api-access-token]  |\n\n\n> **Note:** A protected variable will not show in the CI logs, which adds an\nextra layer of security. If you choose to protect your variables, you need\nto make sure that your `main` branch and the tag `v*` wildcard are protected\nas well.\n\n\n## 2. Add a `config.yml` to your project repository\n\n\nThis file may already exist, but `config.yml` needs to have the following to\nproperly map the variables from step 1 with your Shopify theme for\ndeployments.\n\n\n```yml\n\nstaging:\n  password: ${SHOPIFY_API_ACCESS_TOKEN}\n  theme_id: ${STAGING_THEME_ID}\n  store: ${SHOP_WEB_ADDRESS}\n\nproduction:\n  password: ${SHOPIFY_API_ACCESS_TOKEN}\n  theme_id: ${PRODUCTION_THEME_ID}\n  store: ${SHOP_WEB_ADDRESS}\n```\n\n\n## 3. Add a `.gitlab-ci.yml` file to your project\n\n\nNow set up your pipeline to run on specific triggers. Go to your local theme\nfolder, create a `.gitlab-ci.yml` file at the project root, and add the\nsnippet below. This snippet is the configuration for the CI pipeline.\n\n\n```yml\n\nimage: python:2\n\n\nstages:\n  - staging\n  - production\n\nstaging:\n  image: python:2\n  stage: staging\n  script:\n    - curl -s https://shopify.github.io/themekit/scripts/install.py | python\n    - theme deploy -e=staging\n  only:\n    variables:\n      - $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH\n\nproduction:\n  image: python:2\n  stage: production\n  script:\n    - curl -s https://shopify.github.io/themekit/scripts/install.py | python\n    - theme deploy -e=production --allow-live\n  only:\n    - tags\n\n```\n\n\nIt has two stages: **staging** and **production**. Each will install the\nThemeKit CLI first and then deploy the repository to the corresponding\ntheme.\n\n\n## 4. Now push some changes to deploy\n\n\nAny code pushed to the `main` branch will set up a deployment to the staging\ntheme in Shopify\n\n\n```\n\ngit commit -am \"commit message\"\n\ngit push\n\n```\n\n\nWhen you are ready to push changes to production, add a tag and push it.\n\n\n```\n\ngit tag -a \"v1.0.0\" -m \"First release to production from GitLab\"\n\ngit push --tags\n\n```\n\n\n> **Alternative option:** [Create a tag from\nGitLab](https://docs.gitlab.com/ee/user/project/releases/#create-a-release-in-the-tags-page)\n\n\nThat's it! You're now using CI to automate deployments from GitLab to your\nShopify themes.\n\n\nFurther refine this workflow by [incorporating merge requests\napprovals](/blog/feature-highlight-merge-request-approvals/), [setting up\nmerge trains](/blog/merge-trains-explained/), or learning more\nabout [GitLab CI/CD pipelines](/blog/guide-to-ci-cd-pipelines/).\n\n\nBig thanks to Alex Gogl for their\n[blog](https://medium.com/@gogl.alex/how-to-deploy-shopify-themes-automatically-1ac17ee1229c).\nThis `.gitlab-ci.yml` will appear as an available template when [merge\nrequest !52279](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52279)\nis merged.\n\n\nCover image by [Artem\nGavrysh](https://unsplash.com/@tmwd?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non [Unsplash](https://unsplash.com/)\n",[9],{"slug":1588,"featured":6,"template":700},"deploy-shopify-themes-with-gitlab","content:en-us:blog:deploy-shopify-themes-with-gitlab.yml","Deploy Shopify Themes With Gitlab","en-us/blog/deploy-shopify-themes-with-gitlab.yml","en-us/blog/deploy-shopify-themes-with-gitlab",{"_path":1594,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1595,"content":1601,"config":1609,"_id":1611,"_type":13,"title":1612,"_source":15,"_file":1613,"_stem":1614,"_extension":18},"/en-us/blog/deploying-application-eks",{"title":1596,"description":1597,"ogTitle":1596,"ogDescription":1597,"noIndex":6,"ogImage":1598,"ogUrl":1599,"ogSiteName":685,"ogType":686,"canonicalUrls":1599,"schema":1600},"Deploying apps to GitLab-managed Amazon EKS with Auto DevOps","A Kubernetes tutorial: Use GitLab AutoDevOps to deploy your applications to Amazon EKS.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666959/Blog/Hero%20Images/gitlab-aws-cover.png","https://about.gitlab.com/blog/deploying-application-eks","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to deploy your application to a GitLab-managed Amazon EKS cluster with Auto DevOps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2020-05-05\",\n      }",{"title":1602,"description":1597,"authors":1603,"heroImage":1598,"date":1605,"body":1606,"category":785,"tags":1607},"How to deploy your application to a GitLab-managed Amazon EKS cluster with Auto DevOps",[1604],"Abubakar Siddiq Ango","2020-05-05","\n\nDeploying an application onto Amazon EKS doesn't have to be painful. In fact, GitLab's [Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/) function makes it easy for developers to deploy applications from GitLab onto any cloud. In this tutorial, I break down how to deploy a simple ruby Hello, World application onto our GitLab-managed Amazon EKS cluster, which we created earlier ([read part one to learn how](/blog/gitlab-eks-integration-how-to/)). For the tutorial, I integrated GitLab with Amazon EKS in a GitLab group I created purposely for this, so all the projects created in the group can use the integration without any extra configuration. \n\nIn the previous blog post, we saw how seamless it is to create a Kubernetes cluster on Amazon EKS in GitLab with the right permissions. Developer productivity is greatly improved because there is no more need to manually set-up clusters and the same cluster can be used for multiple projects when Amazon EKS is integrated with GitLab at the group and instance levels, thus making onboarding new projects a breeze.\n\nIn this tutorial, we will be deploying a simple ruby Hello World application to our GitLab-managed Amazon EKS cluster. For the purpose of this tutorial, I have integrated GitLab with Amazon EKS at the group level on a group I own on GitLab.com, this way all projects created in the group can make use of the integration with no extra configuration.\n\n## A few things to note about AutoDevOps\n\nAuto DevOps provides pre-defined [CI/CD configuration](/topics/ci-cd/) which allows you to automatically detect, build, test, deploy, and monitor your applications. All you need to do is push your code and GitLab does the rest, saving you a lot of effort to set up the workflow and processes required to build, deploy, and monitor your project.\n\nYou'll need to execute the following steps for GitLab AutoDevOps to work seamlessly:\n\n* A [base domain](https://docs.gitlab.com/ee/user/project/clusters/#base-domain) name needs to be provided on GitLab’s integration page for Amazon EKS.\n\n ![AutoDevOps Base Domain](https://about.gitlab.com/images/blogimages/deploying-application-eks/base-domain.png){: .shadow.medium.center}\n Setting the base domain for Auto DevOps\n{: .note.text-center}\n\n* GitLab creates subdomains for every project that is deployed using the project slug, project ID and the base domain name. For example, the link `https://abubakar-te-demos-minimal-ruby-app-2.eksdemo-project.gitlabtechevangelism.net/` is automatically created where `abubakar-te-demos-minimal-ruby-app` is the project slug and the project ID of two, both prepended to the base domain name, `eksdemo-project.gitlabtechevangelism.net`.\n\n* Create a wildcard A-record for the base domain and point it to the Ingress endpoint created during the integration in the public-hosted zone of your domain name on Route53. Selecting the ALIAS option in Route 53 will present a list of resources you have already created. You will see your Ingress endpoint in the list of elastic load balancers. Alternatively, you can copy and paste from GitLab’s integration page.\n\n ![Route53 Alias for base Domain](https://about.gitlab.com/images/blogimages/deploying-application-eks/route53.png){: .shadow.small.center}\n Set-up alias for base domain using the generated Ingress endpoint.\n{: .note.text-center}\n\n* Install the pre-defined Kubernetes certificate management controller, certmanager on the GitLab - EKS integration, to ensure every URL created for your application has a Let’s Encrypt certificate.\n\n## Now, lets deploy our application\n\n### How to set-up the project\n\nIt takes five simple steps to set-up the project for your application.\n\nFirst, create a GitLab project from an existing sample, in this case, GitLab’s Auto DevOps example called Minimal Ruby App. There is nothing special about this application, it's just a ruby application you can use to try out the integration. If you integrated Amazon EKS at the group level on GitLab, you can just go ahead to create the project in the group. At the project level, you will have to perform the integration after creating the project.\n\nNext, copy the URL from the “Clone with HTTPS” field of the sample project, Minimal Ruby App:\n\n  ![Cloning over HTTPS](https://about.gitlab.com/images/blogimages/deploying-application-eks/https-clone.png){: .shadow.small.center}\n  The clone sample project.\n{: .note.text-center}\n\nThird, click the \"import project\" tab on the new project page, then click on the \"repo by URL\" button. Paste the URL you copied earlier in the text box for \"Git repository URL\" and click on \"create project\"\n\n  ![Importing Project](https://about.gitlab.com/images/blogimages/deploying-application-eks/import-project.png){: .shadow.medium.center}\n  The progress of the sample project import.\n  {: .note.text-center}\n\nNext, the project will be imported and all the files from the sample will be available in your new project.\n\n  ![Project import progress](https://about.gitlab.com/images/blogimages/deploying-application-eks/import-progress.png){: .shadow.medium.center}\n  The project import is completed.\n  {: .note.text-center}\n\nFinally, go to project settings > CI/CD > Auto DevOps and enable “Default to Auto DevOps pipeline”\n\n  ![Project Settings](https://about.gitlab.com/images/blogimages/deploying-application-eks/project-settings.png){: .shadow.medium.center}\n  Enable the Auto DevOps pipeline.\n  {: .note.text-center}\n\n### How to deploy your application\n\n* Now a pipeline is created and the project built, tested and deployed to production using the [default AutoDevOps CI files](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml).\n\n  ![Project Pipeline](https://about.gitlab.com/images/blogimages/deploying-application-eks/pipeline.png)\n  The first Auto DevOps pipeline.\n  {: .note.text-center}\n\n* Look inside the pipeline output to see the \"deployment to production\" line. This is where the URL is to access your application.\n\n  ![Deployment to production](https://about.gitlab.com/images/blogimages/deploying-application-eks/production-deploy.png)\n  Next, link to the deployed application.\n  {: .note.text-center}\n\n* In the image above, you can see the application has been deployed and can be accessed at `https://abubakar-te-demos-minimal-ruby-app-1.eksdemo-project.gitlabtechevangelism.net/`\n\nAnd it should show a “Hello World” message:\n\n  ![Deployed Application](https://about.gitlab.com/images/blogimages/deploying-application-eks/hello-world.png){: .shadow.medium.center}\n  The deployed application with \"Hello World\" message.\n  {: .note.text-center}\n\n## How to make changes to the deployed application\n\nIf any new changes are pushed, a different set of jobs is run to build, test, and review the changes before they can be merged to the master branch. I changed the \"Hello World\" text in the previous deployment to an HTML text in a new Git branch called `amazon-eks-html` using the GitLab WebIDE tool, and committed the changes.\n\n  ![Make changes to application](https://about.gitlab.com/images/blogimages/deploying-application-eks/new-commit.png)\n  Making new changes to application.\n  {: .note.text-center}\n\nWhile committing the changes, I selected \"start a new merge request (MR),\" which took me to the MR page where I added more information about the changes in a new MR.\n\n  ![New Merge request](https://about.gitlab.com/images/blogimages/deploying-application-eks/new-mr.png)\n  The MR to deploy the new application.\n  {: .note.text-center}\n\nIn the image above, you can see a pipeline is created to build, test and deploy using [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/) to allow you review the changes before deploying to production.\n\n  ![New MR pipeline test](https://about.gitlab.com/images/blogimages/deploying-application-eks/new-mr-test.png)\n  MR with Review Apps\n  {: .note.text-center}\n\nOnce the review is finished, the application is deployed to a dedicated namespace in the Amazon EKS cluster for you to review before deploying to production. A URL for the [Review App](https://docs.gitlab.com/ee/ci/review_apps/) is provided, as shown in the image below.\n\n  ![Review Applications](https://about.gitlab.com/images/blogimages/deploying-application-eks/review-apps.png){: .shadow.medium.center}\n  The application in the Review App.\n  {: .note.text-center}\n\nThe `stop_review` job cleans up the Review App once the review is done. If MR approvals are required, the MR must be approved before being merged into the master branch. Once merged to master, the project is built, tested, and deployed to production.\n\n  ![Merged Change MR](https://about.gitlab.com/images/blogimages/deploying-application-eks/merged-mr.png){: .shadow.medium.center}\n  Deploying changes to production.\n  {: .note.text-center}\n\nThe image above shows that a second pipeline ran after the MR was merged. Once completed, a button is provided to `view app` and also see memory consumption as the app runs. The `view app`\"` button will open the application on the project's subdomain.\n\n  ![Updated application](https://about.gitlab.com/images/blogimages/deploying-application-eks/updated-site.png)\n  Changes deployed to production.\n  {: .note.text-center}\n\n## Deploy to Amazon EKS with Auto DevOps\n\nThe Auto DevOps function at GitLab makes deploying an application to the Amazon EKS cluster quite simple. Really, all you need to do is push code, and Auto DevOps automatically detects the programming language and uses the necessary [buildpack](https://buildpacks.io/) to test, build, and deploy your application. GitLab also takes making changes to your application a step further using Review Apps, which deploys your app to a temporary environment for you to review the app before deploying to production.\n\nIf you have questions about how to integrate GitLab with Amazon EKS to create a Kubernetes cluster, revisit the [first blog post](/blog/gitlab-eks-integration-how-to/).\n",[1241,696,1608,9],"demo",{"slug":1610,"featured":6,"template":700},"deploying-application-eks","content:en-us:blog:deploying-application-eks.yml","Deploying Application Eks","en-us/blog/deploying-application-eks.yml","en-us/blog/deploying-application-eks",{"_path":1616,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1617,"content":1623,"config":1628,"_id":1630,"_type":13,"title":1631,"_source":15,"_file":1632,"_stem":1633,"_extension":18},"/en-us/blog/develop-c-unit-testing-with-catch2-junit-and-gitlab-ci",{"title":1618,"description":1619,"ogTitle":1618,"ogDescription":1619,"noIndex":6,"ogImage":1620,"ogUrl":1621,"ogSiteName":685,"ogType":686,"canonicalUrls":1621,"schema":1622},"Develop C++ unit testing with Catch2, JUnit, and GitLab CI","Learn how to set up, write, and automate C++ unit tests using Catch2 with GitLab CI/CD. See examples from a working air quality app project and AI-powered help from GitLab Duo.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659684/Blog/Hero%20Images/AdobeStock_479904468__1_.jpg","https://about.gitlab.com/blog/develop-c-unit-testing-with-catch2-junit-and-gitlab-ci","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Develop C++ unit testing with Catch2, JUnit, and GitLab CI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fatima Sarah Khalid\"}],\n        \"datePublished\": \"2024-07-02\",\n      }",{"title":1618,"description":1619,"authors":1624,"heroImage":1620,"date":1625,"body":1626,"category":1119,"tags":1627},[1217],"2024-07-02","Continuous integration (CI) and automated testing are important DevSecOps workflows for software developers to detect bugs early, improve code quality, and streamline their development processes. \n\nIn this tutorial, you'll learn how to set up unit testing on a `C++` project with [Catch2](https://github.com/catchorg/Catch2) and GitLab CI for continuous integration. You'll also see how the AI-powered features of [GitLab Duo](https://about.gitlab.com/gitlab-duo/) can help. We’ll use [an air quality monitoring application](https://gitlab.com/gitlab-da/use-cases/ai/ai-applications/air-quality-app) as our reference project.\n\n## Prerequisites\n\n- Ensure you have [CMake](https://cmake.org/ \"CMake\") installed on your machine. \n- A modern `C++` compiler such as GCC or Clang is required. \n- An API key from [OpenWeatherMap](https://openweathermap.org/api) - requires signing up for a free account (1,000/calls per day are included for free). \n\n## Set up the application for testing\n\nThe reference project we’ll be using for demonstrating testing in this blog post is an air quality monitoring application that fetches air quality data from the OpenWeatherMap API based on the U.S zip codes only provided by the user.\n\nHere are the steps to set up the application for testing:\n\n1. Fork the [the reference project](https://gitlab.com/gitlab-da/use-cases/ai/ai-applications/air-quality-app) and clone the fork to your local environment.\n\n2. Generate an API key from  [OpenWeatherMap](https://openweathermap.org/) and export it into the environment. \n\n```shell\nexport API_KEY=\"YOURAPIKEY_HERE\"\n```\n\n3. Alternatively, you can add the key into your `.env` configuration, and source it with `source ~/.env`, or use a different mechanism to populate the environment.\n\n4. Compile and build the project code with the following instructions:\n\n```cpp\ncmake -S . -B build\ncmake --build build\n```\n\n5. Run the application using the executable and passing in a U.S zip code (90210 as an example): \n\n```cpp\n./build/air_quality_app 90210\n```\n\nHere’s an example of what running the program will look like in your terminal:  \n\n```bash\n❯ ./build/air_quality_app 90210\nAir Quality Index (AQI) for Zip Code 90210: 2 (Fair)\n```\n\n## Install Catch2\n\nNow that the application is set up and working, let's start working on adding testing using Catch2. Catch2 is a modern, `C++-native` testing framework for unit tests. \n\nYou can also ask GitLab Duo Chat within your IDE for an introduction to getting started with Catch2 as a `C++` testing framework. GitLab Duo Chat will provide getting started steps as well as an example test: \n\n![GitLab Duo Chat starting steps and example test](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676997/Blog/Content%20Images/1.duo-chat-installing-catch2.png)\n\n1. First navigate to your project’s root directory and create an externals folder using the `mkdir` command.\n\n```shell\nmkdir externals\n```\n\n2. There are several ways to install Catch2 via [its CMake integration](https://github.com/catchorg/Catch2/blob/devel/docs/cmake-integration.md#top). We will use the option of installing it as a submodule and including it as part of the source code to simplify dependency management. To add Catch2 to your project in the `externals` folder: \n\n```shell\ngit submodule add https://github.com/catchorg/Catch2.git externals/Catch2\ngit submodule update --init --recursive\n```\n\n3. Update `CMakeLists.txt` to include Catch2’s directory as a subdirectory. This allows CMake to find and build Catch2 as a part of our project. \n\n```cpp\n# Assuming Catch2 in externals/Catch2\nadd_subdirectory(externals/Catch2)\n```\n\n4. Create a `tests.cpp` file in your project root to write our tests to: \n\n```shell\ntouch tests.cpp\n```\n\n5. Update `CMakeLists.txt` Link against Catch2. When defining your test executable in CMake, link it against Catch2:\n\n```cpp\n# Add tests executable and link it to Catch2\nadd_executable(tests test.cpp)\ntarget_link_libraries(tests PRIVATE Catch2::Catch2WithMain)\n```\n\n## Structure the project for testing\n\nBefore we start writing our tests, we should separate our application logic into separate files in order to maintain and test our code more efficiently. At the end of this section we should have:\n\n```\nmain.cpp containing only the main() function and application setup\nincludes/functions.cpp containing all functional code such as API calls and data processing: \nincludes/functions.h containing the declarations for the functions defined in functions.cpp.  It needs to define the preprocessor macro guards, and include all necessary headers. \n```\n\nApply the following changes to the files: \n\n1. `main.cpp`\n\n```cpp\n#include \u003Ciostream>\n#include \"functions.h\"\n\nint main(int argc, char* argv[]) {\n   if (argc \u003C 2) {\n       std::cerr \u003C\u003C \"Usage: \" \u003C\u003C argv[0] \u003C\u003C \" \u003CZip Code>\" \u003C\u003C std::endl;\n       return 1;\n   }\n\n   std::string zipCode = argv[1];\n   std::string apiKey = getApiKey();\n   if (apiKey.empty()) {\n       std::cerr \u003C\u003C \"API key not found.\" \u003C\u003C std::endl;\n       return 1;\n   }\n\n   auto [lat, lon] = geocodeZipcode(zipCode, apiKey);\n   if (lat == 0 && lon == 0) {\n       std::cerr \u003C\u003C \"Failed to geocode zipcode.\" \u003C\u003C std::endl;\n       return 1;\n   }\n\n   std::string response = fetchAirQuality(lat, lon, apiKey);\n   std::string airQualityInfo = parseAirQualityResponse(response);\n\n   std::cout \u003C\u003C \"Air Quality Index for Zip Code \" \u003C\u003C zipCode \u003C\u003C \": \" \u003C\u003C airQualityInfo \u003C\u003C std::endl;\n\n   return 0;\n}\n```\n\n2. Create a `functions.h:` in the `includes` folder: \n\n```cpp\n#ifndef FUNCTIONS_H\n#define FUNCTIONS_H\n\n#include \u003Cstring>\n#include \u003Cutility>\n#include \u003Cvector>\n\n// Declare the function prototype\nstd::string httpRequest(const std::string& url);\nbool loadEnvFile(const std::string& filename);\nstd::string getApiKey();\nstd::pair\u003Cdouble, double> geocodeZipcode(const std::string& zipCode, const std::string& apiKey);\nstd::string fetchAirQuality(double lat, double lon, const std::string& apiKey);\nstd::string parseAirQualityResponse(const std::string& response);\n\n#endif\n```\n\n3. Create a `functions.cpp` in the `includes` folder: \n\n```cpp\n#include \"functions.h\"\n#include \u003Cfstream>\n#include \u003Celnormous/HTTPRequest.hpp>\n#include \u003Cnlohmann/json.hpp>\n#include \u003Ciostream>\n#include \u003Ccstdlib> // For getenv\n\nstd::string httpRequest(const std::string& url) {\n   try {\n       http::Request request{url};\n       const auto response = request.send(\"GET\");\n       return std::string{response.body.begin(), response.body.end()};\n   } catch (const std::exception& e) {\n       std::cerr \u003C\u003C \"Request failed, error: \" \u003C\u003C e.what() \u003C\u003C std::endl;\n       return \"\";\n   }\n}\nstd::string getApiKey() {\n   const char* envApiKey = std::getenv(\"API_KEY\");\n   if (envApiKey) {\n       return std::string(envApiKey);\n   }\n   // If the environment variable is not set, fallback to the config file\n   std::ifstream configFile(\"config.txt\");\n   std::string line;\n   if (getline(configFile, line)) {\n       return line.substr(line.find('=') + 1);\n   }\n   return \"\";\n}\n\nstd::pair\u003Cdouble, double> geocodeZipcode(const std::string& zipCode, const std::string& apiKey) {\n   std::string url = \"http://api.openweathermap.org/geo/1.0/zip?zip=\" + zipCode + \",US&appid=\" + apiKey;\n   std::string response = httpRequest(url);\n   try {\n       auto json = nlohmann::json::parse(response);\n       if (json.contains(\"lat\") && json.contains(\"lon\")) {\n           double lat = json[\"lat\"];\n           double lon = json[\"lon\"];\n           return {lat, lon};\n       } else {\n           std::cerr \u003C\u003C \"Geocode response missing 'lat' or 'lon' fields: \" \u003C\u003C response \u003C\u003C std::endl;\n       }\n   } catch (const nlohmann::json::parse_error& e) {\n       std::cerr \u003C\u003C \"Failed to parse geocode response: \" \u003C\u003C e.what() \u003C\u003C \" - Response: \" \u003C\u003C response \u003C\u003C std::endl;\n   }\n   return {0, 0};\n}\n\nstd::string fetchAirQuality(double lat, double lon, const std::string& apiKey) {\n   std::string url = \"http://api.openweathermap.org/data/2.5/air_pollution?lat=\" + std::to_string(lat) + \"&lon=\" + std::to_string(lon) + \"&appid=\" + apiKey;\n   std::string response = httpRequest(url);\n   return response;\n}\n\nstd::string parseAirQualityResponse(const std::string& response) {\n   try {\n       auto json = nlohmann::json::parse(response);\n       if (json.contains(\"list\") && !json[\"list\"].empty() && json[\"list\"][0].contains(\"main\")) {\n           int aqi = json[\"list\"][0][\"main\"][\"aqi\"];\n           std::string aqiCategory;\n           switch (aqi) {\n               case 1:\n                   aqiCategory = \"Good\";\n                   break;\n               case 2:\n                   aqiCategory = \"Fair\";\n                   break;\n               case 3:\n                   aqiCategory = \"Moderate\";\n                   break;\n               case 4:\n                   aqiCategory = \"Poor\";\n                   break;\n               case 5:\n                   aqiCategory = \"Very Poor\";\n                   break;\n               default:\n                   aqiCategory = \"Unknown\";\n                   break;\n           }\n           return std::to_string(aqi) + \" (\" + aqiCategory + \")\";\n       } else {\n           return \"No AQI data available\";\n       }\n   } catch (const std::exception& e) {\n       std::cerr \u003C\u003C \"Failed to parse JSON response: \" \u003C\u003C e.what() \u003C\u003C std::endl;\n       return \"Error parsing AQI data\";\n   }\n}\n\n```\n\n4. Now that we have separated the source files, we also need to update our `CMakeLists.txt` to include `functions.cpp` in the `add_executable()` calls:\n\n```cpp\ncmake_minimum_required(VERSION 3.14)\nproject(air-quality-app)\n\n# Set the C++ standard for the project\nset(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED ON)\nset(CMAKE_CXX_EXTENSIONS OFF)\n\ninclude_directories(${CMAKE_SOURCE_DIR}/includes)\n\n# Define the main program executable\nadd_executable(air_quality_app main.cpp includes/functions.cpp)\n\n# Assuming Catch2 in externals/Catch2\nadd_subdirectory(externals/Catch2)\n\n# Add tests executable and link it to Catch2\nadd_executable(tests tests.cpp includes/functions.cpp)\ntarget_link_libraries(tests PRIVATE Catch2::Catch2WithMain)\n```\n\nTo verify that the changes are working, regenerate the CMake configuration and rebuild the source code with the following commands. The build will take longer now that we're compiling Catch2 files. \n\n```shell\nrm -rf build # delete existing build files\ncmake -S . -B build \ncmake --build build  \n```\n\nYou should be able to run the application without any errors.\n\n```shell\n./build/air_quality_app 90210\n```\n\n## Write tests in Catch2  \n\nCatch2 tests are made up of [macros and assertions](https://github.com/catchorg/Catch2/blob/devel/docs/assertions.md). Macros in Catch2 are used to define test cases and sections within those test cases. They help in organizing and structuring the tests. Assertions are used to verify that the code behaves as expected. If an assertion fails, the test case will fail, and Catch2 will report the failure.\n\nLet’s review a basic test scenario for an addition function to understand. Note: This test is read-only, as an example. \n\n```cpp\nint add(int a, int b) {\n   return a + b;\n}\n\nTEST_CASE(\"Addition works correctly\", \"[math]\") {\n   REQUIRE(add(1, 1) == 2);  // Test passes if 1+1 equals 2\n   REQUIRE(add(2, 2) != 5);  // Test passes if 2+2 does not equal 5\n}\n```\n\n- Each test begins with the `TEST_CASE` macro, which defines a test case container. The macro accepts two parameters: a string describing the test case and optionally a second string for tagging the test for easy filtering.\n- Tests are also composed of assertions, which are statements that check if conditions are true. Catch2 provides macros for assertion that include `REQUIRE`, which aborts the current test if the assertion fails, and `CHECK`, which logs the failure but continues with the current test.\n\n### Prepare to write tests with Catch2\n\nTo test the API retrieval functions in our air quality application, we’ll be using mock API requests. Mock API testing is a technique used to test how your application will interact with an external API without making any real API calls. Instead of sending requests to a live API server, we can simulate the responses using predefined data. Mock requests allow us to control the input data and specify exactly what the API would return for different requests, making sure that our tests aren't affected by changes in the real API responses or unexpected data. This also makes it easier for us to simulate and catch different failures.\n\nIn our `tests.cpp` file, let’s define the following function to run mock API requests.   \n\n```cpp\n#include \"includes/functions.h\"\n#include \u003Ccatch2/catch_test_macros.hpp>\n#include \u003Cstring>\n\n// Mock HTTP request function that simulates API responses\nstd::string mockHttpRequest(const std::string& url) {\n   if (url.find(\"geo\") != std::string::npos) {\n       // Mock response for geocoding\n       return R\"({\"lat\": 40.7128, \"lon\": -74.0060})\"; \n   } else if (url.find(\"air_pollution\") != std::string::npos) {\n       // Mock response for air quality\n       return R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\";\n   }\n   // Default mock response for unmatched endpoints\n   return \"{}\";\n}\n// Overriding the actual httpRequest function with the mockHttpRequest for testing\nstd::string httpRequest(const std::string& url) {\n   return mockHttpRequest(url);\n}\n```\n\n- This function simulates HTTP requests and returns predefined JSON responses based on the URL given as input. \n- It also checks the URL to determine which type of data is being requested based on the functionality of the application (geocoding, air pollution, or forecast data). If the URL doesn’t match the expected endpoint, it returns an empty JSON object. \n\nDon't compile the code just yet, as you'll see a linker error. Since we're overriding the original `httpRequest` function with our mock function for testing, we'll need a preprocessor macro to enable conditional compilation - indicating which `httpRequest` function should run when we're compiling tests. \n\n#### Define a preprocessor macro for testing  \n\nBecause we’ve overridden `httpRequest` in our `tests.cpp`, we need to exclude that code from `functions.cpp` when we’re testing. When building tests, we may need to ensure that certain parts of our code behave differently or are excluded. We can do this by defining a preprocessor macro `TESTING` which enables conditional compilation, allowing us to selectively include or exclude code when compiling the test target:  \n\nWe define the `TESTING` macro in our `CMakeLists.txt` at the end:  \n\n```cpp\n# Define TESTING macro for this target\ntarget_compile_definitions(tests PRIVATE TESTING)\n```\n\nAnd add the macro wrapper in  `functions.cpp` around the original `httpRequest` function:  \n\n```cpp\n#ifndef TESTING  // Exclude this part when TESTING is defined\nstd::string httpRequest(const std::string& url) {\n   try {\n       http::Request request{url};\n       const auto response = request.send(\"GET\");\n       return std::string{response.body.begin(), response.body.end()};\n   } catch (const std::exception& e) {\n       std::cerr \u003C\u003C \"Request failed, error: \" \u003C\u003C e.what() \u003C\u003C std::endl;\n       return \"\";\n   }\n}\n#endif\n```\n\nRegenerate the CMake configuration and rebuild the source code to verify it works.\n\n```shell\ncmake --build build  \n```\n\n### Write the first tests \n\nNow, let’s write some tests for our air quality application.\n\n#### Test 1: Verify API key retrieval \n\nThis test ensures that the `getApiKey` function retrieves the API key correctly from the environment variable or the configuration file. Add the test case to our `tests.cpp`:\n\n```cpp\n\nTEST_CASE(\"API Key Retrieval\", \"[api]\") {\n   // Set the API_KEY environment variable for testing\n   setenv(\"API_KEY\", \"test_key\", 1);\n   // Test if the key is retrieved correctly\n   REQUIRE(getApiKey() == \"test_key\");\n}\n```\n\nYou can verify that this tests passes by rebuilding the code and running the tests:\n\n```shell\ncmake --build build\n./build/tests\n```\n\n#### Test 2: Geocode the zip code\n\nThis test ensures that the `geocodeZipcode` function returns the correct latitude and longitude for a given zip code using the mock API response function we set up earlier. The  `geocodeZipcode` function is supposed to hit an API that returns geographic coordinates based on a zip code. \n\nIn `tests.cpp`, add this test case for the zip code 90210: \n\n```cpp\nTEST_CASE(\"Geocode Zip code\", \"[geocode]\") {\n   std::string apiKey = \"test_key\";\n   std::pair\u003Cdouble, double> coordinates = geocodeZipcode(\"90210\", apiKey);\n   // Check latitude\n   REQUIRE(coordinates.first == 40.7128);\n   // Check longitude \n   REQUIRE(coordinates.second == -74.0060);\n}\n```\n\nThe purpose of this test is to verify that the function `geocodeZipcode` can correctly parse the latitude and longitude from the API response. By hardcoding the expected response, we ensure that the test environment is controlled and predictable.\n\n #### Test 3: Air quality API test\n\nThis test ensures that the `fetchAirQuality` function correctly fetches air quality data using the mock API response function we set up earlier. It verifies that the function constructs the API request properly, sends it, and accurately parses the air quality index (AQI) from the mock JSON response. This validation helps ensure that the overall process of fetching and interpreting air quality data works as intended.\n\n```cpp\nTEST_CASE(\"Fetch Air Quality\", \"[airquality]\") {\n   std::string apiKey = \"test_key\";\n   double lat = 40.7128;\n   double lon = -74.0060;\n   std::string response = fetchAirQuality(lat, lon, apiKey);\n   // Check the response\n   REQUIRE(response == R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\");\n}\n```\n\n## Build and run the tests\n\nTo  build and compile our application, we'll use the same CMake commands as before:\n\n```cpp\ncmake -S . -B build\ncmake --build build\n\n```\n\nAfter building, we can run our tests by executing the test binary:  \n\n```cpp\n./build/tests\n\n```\n\nRunning this command will execute all defined tests, and you will see output indicating whether each test has passed or failed.\n\n![Output showing pass/fail of tests](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676998/Blog/Content%20Images/2.running-catch2-tests.png)\n\n## Set up GitLab CI/CD\n\nTo automate the testing process each time we push some new code to our repository, let’s set up [GitLab CI/CD](https://about.gitlab.com/topics/ci-cd/). Create a new `.gitlab-ci.yml` configuration file in the root directory. \n\n```yaml\nimage: gcc:latest\n\nvariables:\n GIT_SUBMODULE_STRATEGY: recursive\n\nstages:\n - build\n - test\n\nbefore_script:\n - apt-get update && apt-get install -y cmake\n\ncompile:\n stage: build\n script:\n   - cmake -S . -B build\n   - cmake --build build\n artifacts:\n   paths:\n     - build/\n\ntest:\n stage: test\n script:\n   - ./build/tests --reporter junit -o test-results.xml\n artifacts:\n   reports:\n     junit: test-results.xml\n```\n\nThis CI/CD configuration will compile both the main application and the test suite, then run the tests, generating a JUnit XML report which GitLab uses to display the test results.  \n\n- In `before_script`, we added an installation for `cmake`, and `git submodule sync --recursive` which initializes and updates our submodules (catch2). \n- In the `test` stage, `--reporter junit -o test-results.xml` specifies that the test results should be treated as a JUnit report which allows GitLab CI to display results in the UI. This is super helpful when you have several tests in your application.  \n\nWe also need to [add an environmental variable](https://docs.gitlab.com/ee/ci/variables/#define-a-cicd-variable-in-the-ui) with the `API_KEY` in project settings on GitLab.\n\nDon’t forget to add all new files to Git, and commit and push the changes in a new MR:\n\n```shell\ngit checkout -b tests-catch2-cicd\n\ngit add includes/functions.{h,cpp} tests.cpp .gitlab-ci.yml \ngit add CMakeLists.txt main.cpp \n\ngit commit -vm “Add Catch2 tests and CI/CD configuration”\ngit push \n```\n\n## View the test report\n\nAfter pushing our code changes, we can review the results of our tests in the GitLab UI in the Pipeline view in the `Tests` tab:\n\n![GitLab pipeline view shows test results](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676998/Blog/Content%20Images/2.0-passed-tests-UI.png)\n\n## Simulate a test failure\n\nTo demonstrate how our UI will handle test failures, we can intentionally introduce a bug into our code and observe the resulting behavior. \n\nLet's modify our `parseAirQualityResponse` function to introduce an error. We can change the AQI category for an AQI value of 2 from \"Fair\" to \"Poor.\" This change will cause the related test to fail, allowing us to see the test failure in the GitLab UI.\n\nIn `functions.cpp`, find the `parseAirQualityResponse` function and modify the switch statement for case `2` to set the `Poor` value instead of `Fair`:\n\n```cpp\n               // Intentional bug:\n               case 2:\n                   aqiCategory = \"Poor\";\n                   break;\n```\n\nIn tests.cpp, add a new test case that directly checks the output of the `parseAirQualityResponse` function. This test ensures that the `parseAirQualityResponse` function correctly parses and categorizes the air quality data from the mock API response. This function takes a JSON response, extracts the AQI value, and translates it into a human-readable category.\n\n```cpp\n\nTEST_CASE(\"Parse Air Quality Response\", \"[airquality]\") {\n   std::string mockResponse = R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\";\n   std::string result = parseAirQualityResponse(mockResponse);\n   // This should fail due to the intentional bug\n   REQUIRE(result == \"2 (Fair)\");\n}\n\n```\n\nCommit the changes, and push them into the MR. Open the MR in your browser. \n\nBy introducing an intentional bug in this function, we can see how a test failure is reported in GitLab's pipelines UI. We must add, commit, and push the changes to our repository to view the test failure in the pipeline. \n\n![Simulated test failure](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676998/Blog/Content%20Images/2.1-failed-test-simulation.png)\n\n![Details of the simulated failed test](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676998/Blog/Content%20Images/2.2-failed-test-simulation-details.png)\n\nOnce we've verified this simulated test failure, we can use `git revert` to roll back that commit. \n\n```shell\ngit revert\n```\n\n## Add and test a new feature\n\nLet’s put what you've learned together by creating a new feature in the air quality application and then writing a test for that feature using Catch2. The new feature will fetch the current weather forecast for the provided zip code.\n\nFirst, we'll define a `Weather` struct and add the function prototype in our `functions.h` file (inside the `#endif`):\n\n```cpp\n\nstruct Weather {\n   std::string main;\n   std::string description;\n   double temperature;\n};\n\nWeather getCurrentWeather(const std::string& apiKey, double lat, double lon);\n```\n\nThen, we implement the `getCurrentWeather` function in `functions.cpp`. This function calls the OpenWeatherMap API to retrieve the current weather and parses the JSON response. This code was generated using [GitLab Duo](https://about.gitlab.com/gitlab-duo/). If you start typing `Weather getCurrentWeather(const std::string& apiKey, double lat, double lon) {` to complete the function, GitLab Duo will provide the function contents for you, line by line. \n\n![GitLab Duo completing the function contents](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676998/Blog/Content%20Images/3.get-current-weather-function-completion.png)\n\nHere's what your `getCurrentWeather()` function can look like: \n\n```cpp\n\nWeather getCurrentWeather(const std::string& apiKey, double lat, double lon) {\n   std::string url = \"http://api.openweathermap.org/data/2.5/weather?lat=\" + std::to_string(lat) + \"&lon=\" + std::to_string(lon) + \"&appid=\" + apiKey;\n   std::string response = httpRequest(url);\n   auto json = nlohmann::json::parse(response);\n   Weather weather;\n   if (!json.is_null()) {\n       weather.main = json[\"weather\"][0][\"main\"];\n       weather.description = json[\"weather\"][0][\"description\"];\n       weather.temperature = json[\"main\"][\"temp\"];\n   }\n   return weather;\n}\n```\n\nAnd, finally, we update our `main.cpp` file in the main function to output the current forecast (and converting Kelvin to Celsius for the output):  \n\n```cpp\n   Weather currentWeather = getCurrentWeather(apiKey, lat, lon);\n   if (currentWeather.main.empty()) {\n       std::cerr \u003C\u003C \"Failed to fetch current weather.\" \u003C\u003C std::endl;\n       return 1;\n   }\n\n   std::cout \u003C\u003C \"Current Weather: \" \u003C\u003C currentWeather.main \u003C\u003C \", \" \u003C\u003C currentWeather.description\n       \u003C\u003C \", temperature \" \u003C\u003C currentWeather.temperature - 273.15 \u003C\u003C \" °C\" \u003C\u003C std::endl;\n```\n\nWe can confirm that our new feature is working by building and running the application:  \n\n```shell\ncmake --build build\n./build/air_quality_app \n```\n\nAnd we should see the following output or similar in case the weather is different on the day the code is run :)\n\n```\nAir Quality Index for Zip Code 90210: 2 (Poor)\nCurrent Weather: Clouds, broken clouds, temperature 23.2 °C\n```\n\nWith all new functionality, there should be testing! We can also write a test to check whether the application is fetching and parsing a weather forecast correctly. This test checks that the function returns a list containing the correct number of forecast entries and that each entry has accurate data regarding time and temperature.\n\n```cpp\nTEST_CASE(\"Current Weather functionality\", \"[api]\") {\n   auto weather = getCurrentWeather(\"dummyApiKey\", 40.7128, -74.0060);\n   // Ensure main weather description is not empty\n   REQUIRE_FALSE(weather.main.empty());\n   // Validate that temperature is a reasonable value\n   REQUIRE(weather.temperature > 0); \n}\n```\n\nWe’ll also have to update our `mockHTTPRequest` function in `tests.cpp` to account for this new test. Modify the if-condition with a new else-if branch checking for the `weather` string in the URL:  \n\n```cpp\n// Mock HTTP request function that simulates API responses\nstd::string mockHttpRequest(const std::string &url)\n{\n   if (url.find(\"geo\") != std::string::npos)\n   {\n       // Mock response for geocoding\n       return R\"({\"lat\": 40.7128, \"lon\": -74.0060})\";\n   }\n   else if (url.find(\"air_pollution\") != std::string::npos)\n   {\n       // Mock response for air quality\n       return R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\";\n   }\n   else if (url.find(\"weather\") != std::string::npos)\n   {\n       // Mock response for current weather\n       return R\"({\n          \"weather\": [{\"main\": \"Clear\", \"description\": \"clear sky\"}],\n          \"main\": {\"temp\": 298.55}\n      })\";\n   }\n   return \"{}\";\n}\n```\n\nAnd verify that our tests are working by rebuilding and running our tests:  \n\n```shell\ncmake --build build \n./build/tests\n```\n\nAll tests should pass, including the new one for Current Weather Functionality. \n\n## Optimize tests.cpp with sections\n\nTo better organize our tests as the project grows and categorize each functionality, we can use Catch2’s `SECTION` macro. The `SECTION` macro allows you to define logically separate test scenarios within a single test case, providing a clean way to test different behaviors or conditions without requiring multiple separate test cases or multiple files. This approach keeps related tests bundled together and also improves test maintainability by allowing shared setup code to be executed repeatedly for each section.\n\nSince some of our functionality is preprocessing data to retrieve information, let’s section our tests as such:\n- preprocessing steps: \n\t- API key validation\n\t- geocoding validation\n-  API data retrieval:\n\t- air pollution retrieval \n\t- forecast retrieval\n\nHere’s what our `tests.cpp` will look like if organized by sections: \n\n```cpp\n#include \"functions.h\"\n#include \u003Ccatch2/catch_test_macros.hpp>\n#include \u003Cstring>\n\n// Mock HTTP request function that simulates API responses\nstd::string mockHttpRequest(const std::string &url)\n{\n   if (url.find(\"geo\") != std::string::npos)\n   {\n       // Mock response for geocoding\n       return R\"({\"lat\": 40.7128, \"lon\": -74.0060})\";\n   }\n   else if (url.find(\"air_pollution\") != std::string::npos)\n   {\n       // Mock response for air quality\n       return R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\";\n   }\n   else if (url.find(\"weather\") != std::string::npos)\n   {\n       // Mock response for current weather\n       return R\"({\n          \"weather\": [{\"main\": \"Clear\", \"description\": \"clear sky\"}],\n          \"main\": {\"temp\": 298.55}\n      })\";\n   }\n   return \"{}\";\n}\n\n// Overriding the actual httpRequest function with the mockHttpRequest for testing\nstd::string httpRequest(const std::string &url)\n{\n   return mockHttpRequest(url);\n}\n\n// Preprocessing Steps\nTEST_CASE(\"Preprocessing Steps\", \"[preprocessing]\") {\n   SECTION(\"API Key Retrieval\") {\n       // Set the API_KEY environment variable for testing\n       setenv(\"API_KEY\", \"test_key\", 1);\n       // Test if the key is retrieved correctly\n       REQUIRE_FALSE(getApiKey().empty());\n   }\n\n   SECTION(\"Geocode Functionality\") {\n       std::string apiKey = \"test_key\";\n       std::pair\u003Cdouble, double> coordinates = geocodeZipcode(\"90210\", apiKey);\n       // Check latitude\n       REQUIRE(coordinates.first == 40.7128);\n       // Check longitude \n       REQUIRE(coordinates.second == -74.0060);\n   }\n}\n\n// API Data Retrieval\nTEST_CASE(\"API Data Retrieval\", \"[data_retrieval]\") {\n   SECTION(\"Air Quality Functionality\") {\n       std::string apiKey = \"test_key\";\n       double lat = 40.7128;\n       double lon = -74.0060;\n       std::string response = fetchAirQuality(lat, lon, apiKey);\n       // Check the response\n       REQUIRE(response == R\"({\"list\": [{\"main\": {\"aqi\": 2}}]})\");\n   }\n\n   SECTION(\"Current Weather Functionality\") {\n       auto weather = getCurrentWeather(\"dummyApiKey\", 40.7128, -74.0060);\n       // Ensure main weather description is not empty\n       REQUIRE_FALSE(weather.main.empty());\n       // Validate that temperature is a reasonable value\n       REQUIRE(weather.temperature > 0);\n   }\n}\n```\n\nRebuild the code and run the tests again to verify.\n\n```shell\ncmake --build build \n./build/tests\n```\n\n## Next steps\n\nIn this post, we covered how to integrate unit testing into a `C++` project using Catch2 testing framework and GitLab CI/CD and set up basic tests for our reference air quality application project.\n\nTo explore these concepts further, you can check out the [Catch2 documentation](https://github.com/catchorg/Catch2) and [GitLab's Unit test report examples documentation](https://docs.gitlab.com/ee/ci/testing/unit_test_report_examples.html). \n\nFor an advanced async exercise, you could build upon this project by using GitLab Duo to implement a feature that retrieves and analyzes historical air quality data and add code quality checks into the CI/CD pipeline. Happy coding! \n",[9,1057,787,720,851],{"slug":1629,"featured":90,"template":700},"develop-c-unit-testing-with-catch2-junit-and-gitlab-ci","content:en-us:blog:develop-c-unit-testing-with-catch2-junit-and-gitlab-ci.yml","Develop C Unit Testing With Catch2 Junit And Gitlab Ci","en-us/blog/develop-c-unit-testing-with-catch2-junit-and-gitlab-ci.yml","en-us/blog/develop-c-unit-testing-with-catch2-junit-and-gitlab-ci",{"_path":1635,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1636,"content":1642,"config":1650,"_id":1652,"_type":13,"title":1653,"_source":15,"_file":1654,"_stem":1655,"_extension":18},"/en-us/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd",{"title":1637,"description":1638,"ogTitle":1637,"ogDescription":1638,"noIndex":6,"ogImage":1639,"ogUrl":1640,"ogSiteName":685,"ogType":686,"canonicalUrls":1640,"schema":1641},"GitLab Duo: AI-powered CI/CD pipeline root cause analysis","Discover how we've infused Root Cause Analysis with AI to help remedy broken CI/CD pipelines, including example scenarios and take-away exercises.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097321/Blog/Hero%20Images/Blog/Hero%20Images/blog-hero-banner-1-0178-820x470-fy25_7JlF3WlEkswGQbcTe8DOTB_1750097321081.png","https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Developing GitLab Duo: Blending AI and Root Cause Analysis to fix CI/CD pipelines\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Rutvik Shah\"},{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2024-06-06\",\n      }",{"title":1643,"description":1638,"authors":1644,"heroImage":1639,"date":1647,"body":1648,"category":718,"tags":1649},"Developing GitLab Duo: Blending AI and Root Cause Analysis to fix CI/CD pipelines",[1645,1646],"Rutvik Shah","Michael Friedrich","2024-06-06","___Generative AI marks a monumental shift in the software development industry, making it easier to develop, secure, and operate software. Our new blog series, written by our product and engineering teams, gives you an inside look at how we create, test, and deploy the AI features you need integrated throughout the enterprise. Get to know new capabilities within GitLab Duo and how they will help DevSecOps teams deliver better results for customers.___\n\nHave you ever encountered a broken [CI/CD](https://about.gitlab.com/topics/ci-cd/benefits-continuous-integration/) pipeline and had to halt your DevSecOps workflow, or even delay software deployment, as you try to figure out the root cause? Traditionally, when something goes wrong in the process of creating software, developers have to troubleshoot, dig through log files, and often do a lot of trial and error development. [GitLab Duo Root Cause Analysis](https://about.gitlab.com/gitlab-duo/), part of our suite of AI-powered features, removes the guesswork by determining the root cause for a failed CI/CD pipeline. In this article, you'll learn what Root Cause Analysis is and how to apply the AI-powered GitLab Duo feature to your DevSecOps workflow.\n\n> Discover the future of AI-driven software development with our GitLab 17 virtual launch event. [Watch today!](https://about.gitlab.com/seventeen/)\n\n### What is Root Cause Analysis?\n\nGitLab Duo Root Cause Analysis is an AI-powered feature that assists you in determining a root cause and suggesting a fix for a CI/CD job log failure by analyzing the logs.\n\nWhile Root Cause Analysis is often seen in product incident management, its workflows and debugging practices can be found in any DevSecOps workflow. Ops teams, administrators, and platform engineers are challenged by infrastructure-as-code (IaC) deployment errors, Kubernetes and GitOps problems, and long stack traces while investigating pipeline failures.\n\nGitLab Duo Root Cause Analysis keeps everyone in the same interface and uses AI-powered help to summarize, analyze, and propose fixes so that organizations can release secure software faster.\n\nA pipeline can encounter failures for a variety of reasons, including syntax errors in the code, missing dependencies that the pipeline relies on, test failures during the build process, Kubernetes and IaC deployment timeouts, and numerous other potential issues. When such failures occur, it becomes the responsibility of everyone to meticulously review the logs generated by the pipeline. This job log review process involves scrutinizing the detailed output to identify the specific errors and pinpoint the root cause of the pipeline failure. For example, the following pipeline has multiple job failures that need to be investigated and fixed.\n\n![Image depicting multiple job failures](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097332/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097332601.png)\n\nThe duration required to fix these failures can vary significantly and is largely influenced by several factors such as:\n- the developer's familiarity with the project\n- their level of experience in dealing with similar issues\n- their overall skill level in troubleshooting and problem-solving within the context of the pipeline.\n\nManual analysis can be exceedingly challenging and time-consuming, given that log data consists of application logs and system messages with a wide variety of potential sources of failures. A typical pipeline fix can consist of several iterations and context switching. The complexity and the unstructured nature of the logs is a perfect fit for speeding up the task using generative AI.  Using AI can reduce the time to identify and fix a pipeline error significantly and also lower the barrier of expertise that would be needed to fix a pipeline such as the above.\n\nWatch GitLab Duo Root Cause Analysis in action:\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n\n \u003Ciframe src=\"https://www.youtube.com/embed/sTpSLwX5DIs?si=J6-0Bf6PtYjrHX1K\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n### How does Root Cause Analysis work?\n\n[Root Cause Analysis](https://docs.gitlab.com/ee/user/ai_experiments.html#root-cause-analysis) works by forwarding a portion of the CI/CD job log to the [GitLab AI Gateway](https://docs.gitlab.com/ee/architecture/blueprints/ai_gateway/). GitLab ensures that the portion sent will fit inside the large language model (LLM) token limits alongside a prompt that has been pre-crafted to provide insights into why the job might have failed. The prompt also instructs the LLM to provide an example of how a user might fix a broken job.\n\nHere are two example scenarios where Root Cause Analysis can provide assistance.\n\n#### 1. Analyze a Python dependency error\n\nA Python application can import package modules with functionality that is not provided in the standard library. The project [Challenge - Root Cause Analysis - Python Config](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/root-cause-analysis/challenge-root-cause-analysis-python-config) implements an application that parses configuration and initializes an SQLite database, which both work well without any dependencies. It uses best practices in CI/CD with a Python environment and caching. The latest feature implementation adds a Redis caching client, and now the CI/CD build is failing for some reason. \n\nBy using Root Cause Analysis, you can immediately learn that the `ModuleNotFoundError` text means that the module is actually not installed in the Python environment. GitLab Duo also suggests an example fix: Installing the Redis module through the PIP package manager. \n\n![Image depicting 'modulenotfounderror' and GL Duo suggested resolution](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097332/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097332602.png)\n\nThe failing pipeline can be viewed [here](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/root-cause-analysis/challenge-root-cause-analysis-python-config/-/jobs/6992716398). \n\nThe Root Cause Analysis prompt provides a summary of the problem, which seems to be a problem with a missing `redis` module. Let's try to fix the problem by installing the `redis` module. You can either call `pip install redis` in the CI/CD job `script` section, or use a more sophisticated approach with the `requirements.txt` file. The latter is useful for a single source of truth for dependencies installed in the development environment and CI/CD pipelines.\n\n```yaml\ntest:\n  extends: [.python-req]\n  stage: test \n  before_script:\n    # [🦊] hint: Root cause analysis.\n    # Solution 1: Install redis using pip\n    - pip install redis\n    # Solution 2: Add redis to requirements.txt, use pip\n    - pip install -r requirements.txt \n\n  script:\n    - python src/main.py\n```\n\nAfter fixing the missing Python dependency, the CI/CD job fails again. Use Root Cause Analysis again to learn that no Redis service is running in the job. Switch to using GitLab Duo Chat and use the prompt `How to start a Redis service in CI/CD` to learn how to configure the `services` attribute in the CI/CD job.\n\n![Depicts the prompt for how to start a Redis service](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097333/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097332602.png)\n\nModify the `.gitlab-ci.yml` with the `test` job, and specify the `redis` service.\n\n```yaml\ntest:\n  extends: [.python-req]\n  stage: test \n  before_script:\n    # [🦊] hint: Root cause analysis.\n    # Solution 1: Install redis using pip\n    - pip install redis\n    # Solution 2: Add redis to requirements.txt, use pip\n    - pip install -r requirements.txt \n\n  script:\n    - python src/main.py\n\n  # Solution 3 - Running Redis\n  services:\n    - redis\n```\n\nRunning the Redis server allows you to successfully execute the Python application, and print its output into the CI/CD job log.\n\n![output of Python application](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097332/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097332603.png)\n\nThe solution is provided in the [solution/ directory](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/root-cause-analysis/challenge-root-cause-analysis-python-config/-/tree/main/solution?ref_type=heads).\n\n**Tip:** You can also ask [GitLab Duo Chat](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html) to follow up on potential future problems:\n\n```markdown\nHow to lint Python code? Which tools are recommended for CI/CD.\nHow to pin a package version in Python requirements file?\t\nWhat are possible ways that this exception stacktrace is triggered in the future?\nAre there ways to prevent the application from failing?\n``` \n\nThe next example is more advanced and includes multiple failures. \n\n#### 2. Analyze missing Go runtime\n\nCI/CD jobs can be executed in containers, spawned from the contributed `image` attribute. If the container does not provide a programming language runtime, the executed `script` sections referencing the `go` binary fail. For example, the error message `/bin/sh: eval: line 149: go: not found` needs to be understood and fixed. \n\nIf the `go` command is not found in the container's runtime context, this can have multiple reasons:\n\n1. The job uses a minimal container image, for example `alpine`, and the Go language runtime was not installed.\n1. The job uses the wrong default container image, for example, specified on top of the CI/CD configuration, or using the `default` keyword.\n1. The job does not use a container image but the shell executor. The host operating system does not have the Go language runtime installed, or it is otherwise broken/not configured.\n\nThe project [Challenge - Root Cause Analysis - Go GitLab Release Fetcher](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/root-cause-analysis/challenge-root-cause-analysis-go-gitlab-release-fetcher) provides an exercise challenge to analyze and fix CI/CD problems with a GitLab release fetcher application, written in Go. The `build` and `docker-build` CI/CD jobs are failing. Fixing the problem requires different scopes: Understanding why the Go runtime is not installed, and learning about the `Dockerfile` syntax. \n\n![Screenshot showing Change Docker Label job failed](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097332/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097332603.png)\n\nThe [`solution/` directory](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/root-cause-analysis/challenge-root-cause-analysis-go-gitlab-release-fetcher) provides two possible solutions after Root Cause Analysis. \n\n## Practice using Root Cause Analysis\n\nHere are some scenarios to use to practice using Root Cause Analysis.\n\n- When you are running into Kubernetes deployment errors or timeouts. \n\n- With OpenTofu or Terraform IaC pipelines failing to provision your cloud resources.\n\n- When the Ansible playbook fails with a cryptic permission error in CI/CD.\n\n- When the Java stack trace is 10 pages long.\n\n- With a shell script highlighting an execution error.\n\n- When a Perl script fails in a single line, which is the only line in the script.\n\n- When the CI/CD job times out and it is unclear which section would cause this.\n\n- When a network connection timeout is reached, and you think it cannot be DNS.\n\n### What is next for GitLab Duo Root Cause Analysis?\n\nWe want to help our users to get their pipelines back to passing in fewer iterations. The Root Cause Analysis will open and show the response in GitLab Duo Chat, our AI assistant. Users can build on the recommendation to generate a more precise fix by asking specific questions (e.g., programming language-specific fixes) or asking for alternative fixes based on the root cause.\n\nFor example, here is the Root Cause Analysis for a failing job:\n\n![Root Cause Analysis response](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097332/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097332603.png)\n\nUsers can ask follow-up questions that build upon the AI-generated response.\n\n- I do not want to create my own Docker image. Please explain different ways to fix the problem.\n\n- I don't have access to the Docker image creation. It seems that the Go binary is missing. Are there alternative images you can suggest?\n\nGitLab also will be running quality benchmarks for the generated responses and shipping usability improvements.\n\nPlease see our [Root Cause Analysis GA epic](https://gitlab.com/groups/gitlab-org/-/epics/13080) for more details. We would also love your feedback on the feature. Please leave a comment on our [Root Cause Analysis feedback issue](https://gitlab.com/groups/gitlab-org/-/epics/13872).\n\n## Get started with Root Cause Analysis\n\nPlease see our [documentation](https://docs.gitlab.com/ee/user/ai_experiments.html#root-cause-analysis) on how to enable the feature available to our GitLab Ultimate customers. Also, GitLab Duo Root Cause Analysis will soon be coming to GitLab self-managed and GitLab Dedicated.\n\nNot a GitLab Ultimate customer? Start [a free trial](https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/blog&glm_content=default-saas-trial) today. \n\n## Read more of our \"Developing GitLab Duo\" series\n\n- [Developing GitLab Duo: How we validate and test AI models at scale](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/)\n\n- [Developing GitLab Duo: AI Impact analytics dashboard measures the ROI of AI](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/)\n\n- [Developing GitLab Duo: How we are dogfooding our AI features](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/)\n\n- [Developing GitLab Duo: Secure and thoroughly test AI-generated code](https://about.gitlab.com/blog/how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code/)",[720,9,851,495,696],{"slug":1651,"featured":90,"template":700},"developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd","content:en-us:blog:developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd.yml","Developing Gitlab Duo Blending Ai And Root Cause Analysis To Fix Ci Cd","en-us/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd.yml","en-us/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd",{"_path":1657,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1658,"content":1664,"config":1670,"_id":1672,"_type":13,"title":1673,"_source":15,"_file":1674,"_stem":1675,"_extension":18},"/en-us/blog/developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities",{"title":1659,"description":1660,"ogTitle":1659,"ogDescription":1660,"noIndex":6,"ogImage":1661,"ogUrl":1662,"ogSiteName":685,"ogType":686,"canonicalUrls":1662,"schema":1663},"Developing GitLab Duo: Use AI to remediate security vulnerabilities ","This tutorial shows how GitLab Duo Vulnerability Explanation and GitLab Duo Vulnerability Resolution, along with our other AI-powered features, can help to address vulnerabilities quickly.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098106/Blog/Hero%20Images/Blog/Hero%20Images/blog-hero-banner-1-0178-820x470-fy25_7JlF3WlEkswGQbcTe8DOTB_1750098106040.png","https://about.gitlab.com/blog/developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Developing GitLab Duo: Use AI to remediate security vulnerabilities \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"},{\"@type\":\"Person\",\"name\":\"Alana Bellucci\"}],\n        \"datePublished\": \"2024-07-15\",\n      }",{"title":1659,"description":1660,"authors":1665,"heroImage":1661,"date":1667,"body":1668,"category":718,"tags":1669},[1646,1666],"Alana Bellucci","2024-07-15","You’ve just started into a new job, and on your first day, a large-scale\nproduction incident requires all hands on deck. There are a number of\ncritical new vulnerabilities that require immediate attention, analysis,\nmitigation and remediation. Where do you start your investigation? \n\n\nLearn how GitLab Duo Vulnerability Explanation and GitLab Duo Vulnerability\nResolution, along with our other AI-powered features, can help you begin\naddressing vulnerabilities in minutes. You will learn how to benefit from\nAI-powered assistance to analyze and explain vulnerabilities in a practical\nexample. Additional remediation is highlighted with AI-generated code fixes\nin MRs to aid faster vulnerability resolution.\n\n\n> Start [a free trial of GitLab\nDuo](https://about.gitlab.com/gitlab-duo/#free-trial) to bring these\npowerful vulnerability remediation benefits to your own organization!\n\n\n## How to get started: Analyze\n\n\nThe first step is to analyze the impact and severity of the vulnerability.\nOpen the GitLab UI and navigate into the [vulnerability\nreport](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/)\nin the `Secure > Vulnerability Report` menu. Filter the vulnerability list\nby `SAST`, and identify the most critical vulnerabilities to work on.\n\n\n![Vulnerability reports\noverview](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/vulnerability_reports_overview_aHR0cHM6_1750098116056.png)\n\n\nThe SAST scanner results are summarized in the detail view, linking to the\nsource code. They provide details from publicly available security\nadvisories. As a developer, it is often hard to start the analysis from the\nsecurity report, unless you are fully aware of the attack scope, technical\ndetails, and vulnerable environments.\n\n\n## Understand and mitigate with Vulnerability Explanation \n\n\nUnderstanding the vulnerability and how to fix it in the best and most\nefficient way is crucial. Fixes must not break existing functionality. If\nthey do, a discussion with maintainers and product owners will be necessary,\nand, as such, will require a high-level summary and potential mitigation\nalternatives. Code that someone who left the company wrote or code that has\nno tests can make the planning for a fix even more difficult. \n\n\nAI-powered Vulnerability Explanation helps with a summary of how an attacker\ncan exploit the vulnerability, and provides more explanations about the\nimpact and potential fixes. \n\n\nThe following example shows an OS Command Injection vulnerability, using\nthis code snippet:\n\n\n```php\n\n\u003C?php \n\n\n// Read variable name from GET request\n\n$name = $_GET['name'];\n\n\n// Use the variable name to call eval and print its value \n\neval('echo $' . $name . ';');\n\n```\n\n\nThe vulnerability report does not go into much detail, and requires\nunderstanding of the full context and impact. Select `Explain vulnerability`\nfrom the upper right corner, which will open GitLab Duo Chat with a\npre-defined prompt action. This will give an additional summary of the\nvulnerability, describe how the vulnerability can be exploited, and provide\na suggested fix. \n\n\n![Improper Neutralization of\n\nSpecial Elements used in an OS Command\n\n('OS Command Injection')\n](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098116057.png)\n\n\n### Make Vulnerability Explanation a conversation with context \n\n\nYou’ll also recognize a change in UX: The previous vulnerability explanation\noverlay was replaced with a GitLab Duo Chat workflow. Sometimes, a complex\nvulnerability unfolds into multiple mitigation steps, or unclear source code\npaths.\n\n\nYou can navigate into the source code tree, and continue with the same Chat\ncontext to explain, fix, refactor, and test the code. \n\n\nLet’s try the full workflow with an example in C, where security scanning\ndetected a buffer overflow.\n\n\n1. Open the security vulnerability detail view, and select \"Explain\nvulnerability\" on the button in the upper right. This will open up the Chat\nprompt, providing a summary of the problem, potential attack vectors, and a\nproposed fix.\n\n\n![AI for vulnerabilities - image\n4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750098116059.png)\n\n\n2. Review the proposed fix, and ask Chat in a follow-up prompt to share\nalternative paths, using `Can you show an alternative fix using a different\nfunction`. The idea is to learn about alternative functions to `strcpy()`\nthat can be more safe to use. \n\n\n![AI for vulnerabilities - image\n3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098116060.png)\n\n\n3. Chat comes up with an alternative fix using `strlcpy()` in the following\nexample. The function only copies as many characters as allowed in the\ntarget string, and always terminates the string with null. It also returns\nthe length of the source string to determine whether the string was\ntruncated. \n\n\n![AI for vulnerabilities - image\n5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098116062.png)\n\n\n4. Next, click on the `Location` file URL to jump into the source code view.\nOpen Chat again, and verify that the previous vulnerability explanation\ncontext is still there. As a next step, we want to add tests before\ncontinuing with a proposed fix. This helps to avoid breaking functionality\nor introduce regressions. For example, use this Chat prompt: `Based on the\nvulnerability context and opened source code, how would you add tests for\nit?`.\n\n\n![AI for vulnerabilities - image 7\n](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098116063.png)\n\n\n5. After generating tests (and assuming they were added now), you can also\nask Chat to refactor the source code, using the prompt `Can you refactor the\nsource code too?` in the same session.\n\n\n![AI for vulnerabilities - image\n6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098116063.png)\n\n\nThe workflow shows how to analyze, understand, mitigate, get alternative\napproaches, add tests, and even refactor fixes for vulnerabilities. \n\n\nYou can continue this path using Chat, and then switch into the Web IDE to\nmodify the source code after learning how to do it. Additional continued\nworkflows include committing changes and triggering CI/CD and security scans\nfor the full DevSecOps lifecycle loop. \n\n\n## Remediate with AI-assisted Vulnerability Resolution \n\n\nUnderstanding and mitigating a security vulnerability still requires\nengineering work to create a fix for the problem, run pipelines and security\nscanning in a new merge request again. It can also be necessary to deploy\nthe fixes into a staging environment and test them for a longer period of\ntime.\n\n\nAI can help here with generating a proposed fix based on the provided\ncontext of the vulnerability and source code.\n\n\nTip: Think of the most annoying vulnerability you had to fix in your career,\nand re-create the use case example for your GitLab Duo adoption. The [MITRE\nCWE Top 25 of the most dangerous software\nweaknesses](https://cwe.mitre.org/top25/archive/2023/2023_top25_list.html)\nalso provides a good starting point.  \n\n\nThe following example implements [CWE-328: Use of a weak hash\nfunction](https://cwe.mitre.org/data/definitions/328.html) by using `md5`.\nIt is correctly identified by [SAST\nscanning](https://docs.gitlab.com/ee/user/application_security/sast/). \n\n\n```python\n\nimport hashlib\n\n\nclass User:\n    def __init__(self, username, password):\n        self.username = username\n        self.password = password\n\n    def set_password(self, password):\n        self.password = hashlib.md5(password.encode()).hexdigest()\n```\n\n\n![AI for vulnerabilities - image\n8](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098116064.png)\n\n\nClick on the button in the upper right `Resolve with merge request`.  This\nwill open an MR that uses AI to propose the fix. For this vulnerability, one\npossible fix could be using a different hash function. \n\n\n![AI for vulnerabilities - image\n9](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098116065.png)\n\n\nAnother common vulnerability example is not checking function error codes or\npotential exceptions. The following C code snippets implement an example for\ntiming attacks against file operations with\n[CWE-362](https://cwe.mitre.org/data/definitions/362.html) for the `fopen()`\nand `chmod()` calls. \n\n\n```c\n\n#include \u003Cstdio.h>\n\n#include \u003Cstring.h>\n\n#include \u003Csys/mman.h>\n\n#include \u003Csys/stat.h>\n\n#include \u003Cunistd.h>\n\n\nint main(int argc, char **argv) {\n\n    // File operations\n    char *fname = \"gitlab.keksi\";\n\n    FILE *fp;\n    fp = fopen(fname, \"r\");\n    fprintf(fp, \"Hello from GitLab Duo Vulnerability Resolution Challenge\");\n    fclose(fp);\n\n    // Potential chmod() timing attacks    \n\n    // Make the file world readable\n    chmod(fname, S_IRWXU|S_IRWXG|S_IRWXO);\n\n    return 0;\n}\n\n```\n\n\nThe SAST report for `chmod()` can look like the following: \n\n\n![AI for vulnerabilities - image\n10](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098116065.png)\n\n\nThe proposed `chmod()` merge request includes error handling, and fixes\nanother potential issue with world writable files, changing the permissions\nfrom `777` to `600`.\n\n\n![AI for vulnerabilities - image\n11](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098116/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098116066.png)\n\n\n> Try this async exercise: Find, analyze, and fix the vulnerability for the\n`fopen()` function.\n\n\n## More AI assistance required from GitLab Duo \n\n\nOften, a security problem can be resolved with a quick fix or a workaround\nthat grants the development teams time to discuss and plan a more long-term\nsolution. In other cases, the problem becomes more complex and requires\nfeature APIs disabled, or firewall mitigation, until a proper fix can be\nrolled into production.\n\n\nGitLab Duo offers additional AI-powered features that can help resolve these\nissues. \n\n\n**Code Explanation:** As a developer or security engineer, it's crucial to\nfeel confident in the changes you've made. Within the IDE, you can use the\n[Code Explanation\nfeature](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#explain-code-in-the-ide)\nto gain a deeper understanding of the AI-suggested fix for the\nvulnerability. This ensures you know exactly what adjustments have been made\nand why.\n\n\n**Root Cause Analysis:** If the fix breaks your pipeline, you can utilize\nthe [Root Cause Analysis\nfeature](https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd/).\nThis tool helps identify and explain the underlying problem, allowing you to\naddress it effectively. After applying the necessary corrections, you can\nrerun the tests to ensure a successful resolution.\n\n\n**Refactor:** Even if the vulnerability has been fixed, it's worth\nconsidering if the code can be written in a safer manner. In the IDE, you\ncan open GitLab Duo Chat and use the [refactor\naction](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#refactor-code-in-the-ide)\nto explore alternative, more secure ways to write your code. This proactive\napproach helps maintain a robust and secure codebase.\n\n\nBy leveraging these GitLab Duo features, you can confidently navigate and\nresolve vulnerabilities, ensuring your code remains secure and efficient.\n\n\n## What’s next?\n\n\nWe plan to bring both Vulnerability Explanation and Vulnerability Resolution\n\"left\" by incorporating them directly into the MR process. This integration\nensures that you can address and resolve vulnerabilities earlier in the\ndevelopment cycle, streamlining your workflow and enhancing code security\nfrom the outset.\n\n\n## Get started with GitLab Duo\n\n\nPlease see our\n[documentation](https://docs.gitlab.com/ee/user/gitlab_duo/turn_on_off.html)\non how to enable the feature available to our GitLab Ultimate customers.\nAlso, GitLab Duo [Vulnerability\nExplanation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#explaining-a-vulnerability)\nand [Vulnerability\nResolution](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#vulnerability-resolution)\nwill soon be coming to GitLab self-managed and GitLab Dedicated.\n\n\nYou can keep up with what's new in GitLab Duo by [following the \"Developing\nGitLab Duo\" blog\nseries](https://about.gitlab.com/blog/developing-gitlab-duo-series/).\n\n\n> Start [a free trial of GitLab\nDuo](https://about.gitlab.com/gitlab-duo/#free-trial) to bring these\npowerful vulnerability remediation benefits to your own organization!\n",[720,830,694,696,9],{"slug":1671,"featured":90,"template":700},"developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities","content:en-us:blog:developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities.yml","Developing Gitlab Duo Use Ai To Remediate Security Vulnerabilities","en-us/blog/developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities.yml","en-us/blog/developing-gitlab-duo-use-ai-to-remediate-security-vulnerabilities",{"_path":1677,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1678,"content":1684,"config":1690,"_id":1692,"_type":13,"title":1693,"_source":15,"_file":1694,"_stem":1695,"_extension":18},"/en-us/blog/devops-adoption",{"title":1679,"description":1680,"ogTitle":1679,"ogDescription":1680,"noIndex":6,"ogImage":1681,"ogUrl":1682,"ogSiteName":685,"ogType":686,"canonicalUrls":1682,"schema":1683},"Understand how your teams adopt DevOps with DevOps reports","Learn about analytics, DevOps reports, DevOps scores, and more.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749668473/Blog/Hero%20Images/john-schnobrich-FlPc9_VocJ4-unsplash.jpg","https://about.gitlab.com/blog/devops-adoption","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Understand how your teams adopt DevOps with DevOps reports\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Orit Golowinski\"}],\n        \"datePublished\": \"2021-12-15\",\n      }",{"title":1679,"description":1680,"authors":1685,"heroImage":1681,"date":1687,"body":1688,"category":1119,"tags":1689},[1686],"Orit Golowinski","2021-12-15","\n\nGitLab has an extraordinary range of features for a single application, providing an [entire DevOps platform](/stages-devops-lifecycle/) from [portfolio planning](/stages-devops-lifecycle/plan/) all the way through to [monitoring](/stages-devops-lifecycle/monitor/) and [service desk](https://docs.gitlab.com/ee/user/project/service_desk/). As such, GitLab is uniquely positioned to deliver a complete picture of your organization's DevOps journey and your return on investment in automation and DevOps practices.\n\nSome of the most interesting and difficult questions that organizations ask themselves are:\n\n* What do we gain from different development practices used by our teams?\n* What makes one team more efficient than another?\n* What practices have been successful in one team that we can introduce to others?\n\n## Analytics\n\nGitLab has several metrics to give you insight into the development lifecycle:\n\n* [Application Security](https://docs.gitlab.com/ee/user/application_security/security_dashboard/#project-security-dashboard) -  provides a comprehensive set of features for viewing and managing vulnerabilities.\n* [CI/CD](https://docs.gitlab.com/ee/user/analytics/ci_cd_analytics.html) - tracks the history of your pipeline successes and failures, as well as how long each pipeline ran.\n* [Code Review](https://docs.gitlab.com/ee/user/analytics/code_review_analytics.html) - displays open merge requests and their review time.\n* [Insights](https://docs.gitlab.com/ee/user/project/insights/index.html)- allows you to configure custom analytics that will be displayed.\n* [Issue](https://docs.gitlab.com/ee/user/group/issues_analytics/index.html) - illustrates the number of issues created each month.\n* [Merge Request](https://docs.gitlab.com/ee/user/analytics/merge_request_analytics.html) - displays information that will help you evaluate the efficiency and productivity of your merge request process.\n* [Repository](https://docs.gitlab.com/ee/user/analytics/repository_analytics.html) - displays information such as commit statistics, code coverage, and programming languages used in the repository.\n* [Value Stream Analytics](https://docs.gitlab.com/ee/user/analytics/value_stream_analytics.html) - measures the time spent to go from an idea to production.\n\nSome analytics are only available for instance-level (self-managed), group level, or project level. Read [more](https://docs.gitlab.com/ee/user/analytics/) about analytics.\n\nThese analytics are a great way to see contributions from different projects and groups. On their own, however, they don't give insights into which processes your teams are using. For that, we offer DevOps Reports.\n\n## DevOps adoption reports\n\nDevOps Adoption is a DevOps Report located in group-level analytics. It shows you data for how teams in your organization use the most essential GitLab features.\n\nYou can use DevOps Adoption to:\n\n- Identify specific subgroups that are lagging in their adoption of GitLab features, so you can guide them on their DevOps journey.\n- Find subgroups that have successfully adopted certain features, and could provide guidance to other subgroups on how to use those features.\n- Verify if you are getting the return on investment that you expected from GitLab.\n\n![DevOps Adoption](https://about.gitlab.com/images/blogimages/devops_reports.png){: .shadow}\n\nIn this example, we can see some interesting data on how a team uses features in development, security, and operations categories:\n\n* **Development**\n  * Approvals: At least one merge request approval on a merge request.\n  * Code owners: At least 1 defined code owner that owns a specific file or repository in the group.\n  * Issues: At least 1 issue opened in this group.\n  * Merge requests: At least 1 merge request opened in this group.\n* **Security**\n  * DAST:  At least 1 DAST scan run in a pipeline in the group.\n  * Dependency Scanning: At least 1 dependency scan ran in a pipeline in the group.\n  * Fuzz Testing: At least 1 fuzz testing scan ran in a pipeline in the group.\n  * SAST: At least 1 SAST scan ran in a pipeline in the group.\n* **Operations**\n  * Deployments: At least 1 deployment.\n  * Pipelines: At least 1 pipeline ran successfully.\n  * Runners: At least 1 runner configured for the project or group.\n\nIn the future we plan to add even more feature categories to DevOps Reports, such as:\n* [Environments](https://docs.gitlab.com/ee/ci/environments/#environments-and-deployments)\n* [Pages](https://docs.gitlab.com/ee/user/project/pages/)\n* [Compliance Pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration)\n* [Incidents](https://docs.gitlab.com/ee/operations/incident_management/incidents.html)\n* [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/#review-apps)\n\n...and much more. You can follow our future plans in the following [epic](https://gitlab.com/groups/gitlab-org/-/epics/5019).\n\n_DevOps Reports are available for the Ultimate tier for self-managed and SaaS users. To find DevOps Reports, go to your group and in the left sidebar, select Analytics > DevOps adoption_\n\n## DevOps Score\n\nYou can use the DevOps score to compare your DevOps status to other organizations.\n\nThe DevOps Score tab shows usage of major GitLab features on your instance over the last 30 days. GitLab calculates the averages feature usage based on the number of billable users in that time period. You can also see the Leader usage score, calculated from top-performing instances based on Service Ping data that GitLab collects. GitLab compares your score to the lead score of each feature and shows it as a percentage underneath the feature. Your overall DevOps Score is an average of your feature scores.\n\nTo analyze your DevOps Score, GitLab aggregates Service Ping (sometimes referred to as Usage Ping) data on GitLab servers for analysis. Your usage information is not sent to any other GitLab instances. If you have just started using GitLab, it may take a few weeks for GitLab to collect enough data to calculate your DevOps Score.\n\n![DevOps Score](https://about.gitlab.com/images/blogimages/dev_ops_score_v12_6.png){: .shadow}\n\n_DevOps score is available at the admin panel for all tiers under Analytics > DevOps Reports._\n\nTo see the DevOps score, you must activate your GitLab instance’s [Service Ping](https://docs.gitlab.com/ee/administration/settings/usage_statistics.html#service-ping). This is because DevOps Score is a comparative tool, so your score data must first be centrally processed by GitLab, Inc.\n\nThere are several benefits of enabling Service Ping, such as DevOps Score and cohorts:\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ZhLrhZlb_zI\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n## Cohorts\n\nCohorts shows your teams' GitLab activities over time, and is a useful tool for administrators to view user retention and manage seats in their GitLab instance.\n\n![Cohorts](https://about.gitlab.com/images/blogimages/cohorts_v13_9_a.png){: .shadow}\n\nUsers are considered active if they have performed at least one of the following activities:\n\n* Sign in to GitLab.\n* Perform a Git activity such as push or pull.\n* Visit pages related to dashboards, projects, issues, or merge requests.\n* Use the API.\n* Use the GraphQL API.\n\nCover image credit:\n\nCover image by [John Schnobrich](https://unsplash.com/photos/FlPc9_VocJ4) on [Unsplash](https://unsplash.com)\n{: .note}\n",[789,696,9],{"slug":1691,"featured":6,"template":700},"devops-adoption","content:en-us:blog:devops-adoption.yml","Devops Adoption","en-us/blog/devops-adoption.yml","en-us/blog/devops-adoption",{"_path":1697,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1698,"content":1704,"config":1710,"_id":1712,"_type":13,"title":1713,"_source":15,"_file":1714,"_stem":1715,"_extension":18},"/en-us/blog/devops-workflows-json-format-jq-ci-cd-lint",{"title":1699,"description":1700,"ogTitle":1699,"ogDescription":1700,"noIndex":6,"ogImage":1701,"ogUrl":1702,"ogSiteName":685,"ogType":686,"canonicalUrls":1702,"schema":1703},"JSON formatting and CI/CD linting tips for DevOps workflows","Learn how to filter in JSON data structures and interact with the REST API. Use the GitLab API to lint your CI/CD configuration and dive into Git hooks speeding up your workflows.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749681979/Blog/Hero%20Images/gert-boers-unsplash.jpg","https://about.gitlab.com/blog/devops-workflows-json-format-jq-ci-cd-lint","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tips for productive DevOps workflows: JSON formatting with jq and CI/CD linting automation\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2021-04-21\",\n      }",{"title":1705,"description":1700,"authors":1706,"heroImage":1701,"date":1707,"body":1708,"category":785,"tags":1709},"Tips for productive DevOps workflows: JSON formatting with jq and CI/CD linting automation",[1646],"2021-04-21","## What is JSON linting?\n\n\nTo understand JSON linting, let’s quickly break down the two concepts of\nJSON and linting. \n\n\n***JSON*** is an acronym for JavaScript Object Notation, which is a\nlightweight, text-based, open standard format designed specifically for\nrepresenting structured data based on the JavaScript object syntax. It is\nmost commonly used for transmitting data in web applications. It parses data\nfaster than XML and is easy for humans to read and write.\n\n\n***Linting*** is a process that automatically checks and analyzes static\nsource code for programming and stylistic errors, bugs and suspicious\nconstructs. \n\n\nJSON has become popular because it is human-readable and doesn’t require a\ncomplete markup structure like XML. It is easy to analyze into logical\nsyntactic components, especially in JavaScript. It also has many JSON\nlibraries for most programming languages.\n\n\n### Benefits of JSON linting\n\n\nFinding an error in JSON code can be challenging and time-consuming. The\nbest way to find and correct errors while simultaneously saving time is to\nuse a linting tool. When Json code is copied and pasted into the linting\neditor, it validates and reformats Json. It is easy to use and supports a\nwide range of browsers, so applications development with Json coding don’t\nrequire a lot of effort to make them browser-compatible.\n\n\nJSON linting is an efficient way to reduce errors and it improves the\noverall quality of the JSON code. This can help accelerate development and\nreduce costs because errors are discovered earlier.\n\n\n### Some common JSON linting errors\n\n\nIn instances where a JSON transaction fails, the error information is\nconveyed to the user by the API gateway. By default, the API gateway returns\na very basic fault to the client when a message filter has failed.\n\n\nOne common JSON linting error is parsing. A “parse: unexpected character\"\nerror occurs when passing a value that is not a valid JSON string to the\nJSON. parse method, for example, a native JavaScript object. To solve the\nerror, make sure to only pass valid JSON strings to the JSON.\n\n\nAnother common error is NULL or inaccurate data errors, not using the right\ndata type per column or extension for JSON files, and not ensuring every row\nin the JSON table is in the JSON format.\n\n\n### How to fix JSON linting errors\n\n\nIf you encounter a NULL or inaccurate data error in parsing, the first step\nis to make sure you use the right data type per column. For example, in the\ncase of “age,” use 12 instead of twelve.\n\n\nAlso make sure you are using the right extension for JSON files. When using\na compressed JSON file, it must end with “json” followed by the extension of\nthe format, such as “.gz.”\n\n\nNext, make sure the JSON format is used for every row in the JSON table.\nCreate a table with a delimiter that is not in the input files. Then, run a\nquery equivalent to the return name of the file, row points and the file\npath for the null NSON rows.\n\n\nSometimes you may find files that are not your source code files, but ones\ngenerated by the system when compiling your project. In that instance, when\nthe file has a .js extension, the ESLint needs to exclude that file when\nsearching for errors. One method of doing this is by using ‘IgnorePatterns:’\nin .eslintrc.json file either after or before the “rules” tag.\n\n\n“ignorePatterns”: [“temp.js”, “**/vendor/*.js”],\n\n\n“rules”: {\n\n\nAlternatively, you can create a separate file named‘.eslintignore’ and\nincorporate the files to be excluded as shown below :\n\n**/*.js\n\nIf you opt to correct instead of ignore, look for the error code in the last\ncolumn. Correct all the errors in one fule and rerun ‘npx eslint . >errfile’\nand ensure all the errors of that type are cleared. Then look for the next\nerror code and repeat the procedure until all errors are cleared.\n\n\nOf course, there will be instances when you won’t understand an error, so in\nthat case, open\n[https://eslint.org/docs/user-guide/getting-started](https://eslint.org/docs/user-guide/getting-started)\nand type the error code in the ‘Search’ field on the top of the document.\nThere you will find very detailed instructions as to why that error is\nraised and how to fix it.\n\n\nFinally, you can forcibly fix errors automatically while generating the\nerror list using:\n\n\nNpx eslintrc . — fix \n\n\nThis is not recommended until you become more well-versed with lint errors\nand how to fix them. Also, you should keep a backup of the files you are\nlinting because while fixing errors, certain code may get overwritten, which\ncould cause your program to fail.\n\n\n## JSON linting best practices\n\n\nHere are some tips for helping your consumers use your output:\n\n\nFirst, always enclose the **Key** **:** **Value** pair within **double\nquotes**. It may be convenient (not sure how) to generate with Single\nquotes, but JSON parser don’t like to parse JSON objects with single quotes.\n\n\nFor numerical values, quotes are optional but it is a good idea to enclose\nthem in double quotes.\n\n\nNext, don’t ever use hyphens in your key fields because it breaks python and\nscala parser. Instead use underscores (_). \n\n\nIt’s a good idea to always create a root element, especially when you’re\ncreating a complicated JSON.\n\n\n\nModern web applications come with a REST API which returns JSON. The format\nneeds to be parsed, and often feeds into scripts and service daemons polling\nthe API for automation.\n\n\nStarting with a new REST API and its endpoints can often be overwhelming.\nDocumentation may suggest looking into a set of SDKs and libraries for\nvarious languages, or instruct you to use `curl` or `wget` on the CLI to\nsend a request. Both CLI tools come with a variety of parameters which help\nto download and print the response string, for example in JSON format.\n\n\nThe response string retrieved from `curl` may get long and confusing. It can\nrequire parsing the JSON format and filtering for a smaller subset of\nresults. This helps with viewing the results on the CLI, and minimizes the\ndata to process in scripts. The following example retrieves all projects\nfrom GitLab and returns a paginated result set with the first 20 projects:\n\n\n```shell\n\n$ curl \"https://gitlab.com/api/v4/projects\"\n\n```\n\n\n![Raw JSON as API\nresponse](https://about.gitlab.com/images/blogimages/devops-workflows-json-format-jq-ci-cd-lint/gitlab_api_response_raw_json.png){:\n.shadow}\n\n\nThe [GitLab REST API\ndocumentation](https://docs.gitlab.com/ee/api/#how-to-use-the-api) guides\nyou through the first steps with error handling and authentication. In this\nblog post, we will be using the [Personal Access\nToken](https://docs.gitlab.com/ee/api/#personalproject-access-tokens) as the\nauthentication method. Alternatively, you can use [project access\ntokens](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html)\nfor [automated\nauthentication](https://docs.gitlab.com/ee/api/#authentication) that avoids\nthe use of personal credentials.\n\n\n### REST API authentication\n\n\nSince not all endpoints are accessible with anonymous access they might\nrequire authentication. Try fetching user profile data with this request:\n\n\n```shell\n\n$ curl \"https://gitlab.com/api/v4/user\"\n\n{\"message\":\"401 Unauthorized\"}\n\n```\n\n\nThe API request against the `/user` endpoint requires to pass the personal\naccess token into the request, for example, as a request header. To avoid\nexposing credentials on the terminal, you can export the token and its value\ninto the user's environment. You can automate the variable export with ZSH\nand the [.env\nplugin](https://github.com/ohmyzsh/ohmyzsh/tree/master/plugins/dotenv) in\nyour shell environment. You can also source the `.env` once in the existing\nshell environment.\n\n\n```shell\n\n$ vim ~/.env\n\n\nexport GITLAB_TOKEN=”...”\n\n\n$ source ~/.env\n\n```\n\n\nScripts and commands being run in your shell environment can reference the\n`$GITLAB_TOKEN` variable. Try querying the user API endpoint again, with\nadding the authorization header into the request:\n\n\n```shell\n\n$ curl -H \"Authorization: Bearer $GITLAB_TOKEN\"\n\"https://gitlab.com/api/v4/user\"\n\n```\n\n\nA reminder that only administrators can see the attributes of all users, and\nthe individual can only see their user profile – for example, `email` is\nhidden from the public domain.\n\n\n### How to request responses in JSON\n\n\nThe [GitLab API provides many\nresources](https://docs.gitlab.com/ee/api/api_resources.html) and URL\nendpoints. You can manage almost anything with the API that you’d otherwise\nconfigure using the graphic user interface.\n\n\nAfter sending the [API\nrequest](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_message),\nthe [response\nmessage](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Response_message)\ncontains the body as string, for example as a [JSON content\ntype](https://docs.gitlab.com/ee/api/#content-type). `curl` can provide more\ninformation about the response headers which is helpful for debugging.\nMultiple verbose levels enable the full debug output with `-vvv`:\n\n\n```shell\n\n$ curl -vvv \"https://gitlab.com/api/v4/projects\"\n\n[...]\n\n* SSL connection using TLSv1.2 / ECDHE-RSA-CHACHA20-POLY1305\n\n* ALPN, server accepted to use h2\n\n* Server certificate:\n\n*  subject: CN=gitlab.com\n\n*  start date: Jan 21 00:00:00 2021 GMT\n\n*  expire date: May 11 23:59:59 2021 GMT\n\n*  subjectAltName: host \"gitlab.com\" matched cert's \"gitlab.com\"\n\n*  issuer: C=GB; ST=Greater Manchester; L=Salford; O=Sectigo Limited;\nCN=Sectigo RSA Domain Validation Secure Server CA\n\n*  SSL certificate verify ok.\n\n[...]\n\n> GET /api/v4/projects HTTP/2\n\n> Host: gitlab.com\n\n> User-Agent: curl/7.64.1\n\n> Accept: */*\n\n[...]\n\n\u003C HTTP/2 200\n\n\u003C date: Mon, 19 Apr 2021 11:25:31 GMT\n\n\u003C content-type: application/json\n\n[...]\n\n[{\"id\":25993690,\"description\":\"project for adding\nissues\",\"name\":\"project-for-issues-1e1b6d5f938fb240\",\"name_with_namespace\":\"gitlab-qa-sandbox-group\n/ qa-test-2021-04-19-11-13-01-d7d873fd43cd34b6 /\nproject-for-issues-1e1b6d5f938fb240\",\"path\":\"project-for-issues-1e1b6d5f938fb240\",\"path_with_namespace\":\"gitlab-qa-sandbox-group/qa-test-2021-04-19-11-13-01-d7d873fd43cd34b6/project-for-issues-1e1b6d5f938fb240\"\n\n\n[... JSON content ...]\n\n\n\"avatar_url\":null,\"web_url\":\"https://gitlab.com/groups/gitlab-qa-sandbox-group/qa-test-2021-04-19-11-12-56-7f3128bd0e41b92f\"}}]\n\n* Closing connection 0\n\n```\n\n\nThe `curl` command output provides helpful insights into TLS ciphers and\nversions, the request lines starting with `>` and response lines starting\nwith `\u003C`. The response body string is encoded as JSON.\n\n\n### How to see the structure of the returned JSON\n\n\nTo get a quick look at the structure of the returned JSON file, try these\ntips:\n\n\n* Enclose square brackets to identify an array `[ …. ]`.\n\n* Enclose curly brackets identify a\n[dictionary](https://en.wikipedia.org/wiki/Associative_array) `{ … }`.\nDictionaries are also called associative arrays, maps, etc.\n\n* `”key”: value` indicates a key-value pair in a dictionary, which is\nidentified by curly brackets enclosing the key-value pairs.\n\n\nThe values in [JSON](https://en.wikipedia.org/wiki/JSON) consist of specific\ntypes - a string value is put in double-quotes. Boolean true/false, numbers,\nand floating-point numbers are also present as types. If a key exists but\nits value is not set, REST APIs often return `null`.\n\n\nVerify the data structure by running \"linters\". Python's JSON module can\nparse and lint JSON strings. The example below misses a closing square\nbracket to showcase the error:\n\n\n```shell\n\n$ echo '[{\"key\": \"broken\"}' | python -m json.tool\n\nExpecting object: line 1 column 19 (char 18)\n\n```\n\n\n[jq](https://stedolan.github.io/jq/) – a lightweight and flexible CLI\nprocessor – can be used as a standalone tool to parse and validate JSON\ndata.\n\n\n```shell\n\n$ echo '[{\"key\": \"broken\"}' | jq\n\nparse error: Unfinished JSON term at EOF at line 2, column 0\n\n```\n\n\n[`jq` is available](https://stedolan.github.io/jq/download/) in the package\nmanagers of most operating systems.\n\n\n```shell\n\n$ brew install jq\n\n$ apt install jq\n\n$ dnf install jq\n\n$ zypper in jq\n\n$ pacman -S jq\n\n$ apk add jq\n\n```\n\n\n### Dive deep into JSON data structures\n\n\nThe true power of `jq` lies in how it can be used to parse JSON data:\n\n\n> `jq` is like `sed` for JSON data. It can be used to slice, filter, map,\nand transform structured data with the same ease that `sed`, `awk`, `grep`\netc., let you manipulate text.\n\n\nThe output below shows how it looks to run the request against the project\nAPI again, but this time, the output is piped to `jq`.\n\n\n```shell\n\n$ curl \"https://gitlab.com/api/v4/projects\" | jq\n\n[\n  {\n    \"id\": 25994891,\n    \"description\": \"...\",\n    \"name\": \"...\",\n\n[...]\n\n    \"forks_count\": 0,\n    \"star_count\": 0,\n    \"last_activity_at\": \"2021-04-19T11:50:24.292Z\",\n    \"namespace\": {\n      \"id\": 11528141,\n      \"name\": \"...\",\n\n[...]\n\n    }\n  }\n]\n\n```\n\n\nThe first difference is the format of the JSON data structure, so-called\n[pretty-printed](https://en.wikipedia.org/wiki/Prettyprint). New lines and\nindents in data structure scopes help your eyes and allow you to identify\nthe inner and outer data structures involved. This format is needed to\ndetermine which `jq` filters and methods you want to apply next.\n\n\n#### About arrays and dictionaries\n\n\nThe set of results from an API often is returned as a list (or \"array\") of\nitems. An item itself can be a single value or a JSON object. The following\nexample mimics the response from the GitLab API and creates an array of\ndictionaries as a nested result set.\n\n\n```shell\n\n$ vim result.json\n\n[\n  {\n    \"id\": 1,\n    \"name\": \"project1\"\n  },\n  {\n    \"id\": 2,\n    \"name\": \"project2\"\n  },\n  {\n    \"id\": 3,\n    \"name\": \"project-internal-dev\",\n    \"namespace\": {\n      \"name\": \"🦊\"\n    }\n  }\n]\n\n```\n\n\nUse `cat` to print the file content on stdout and pipe it into `jq`. The\nouter data structure is an array – use `-c .[]` to access and print all\nitems.\n\n\n```shell\n\n$ cat result.json | jq -c '.[]'\n\n{\"id\":1,\"name\":\"project1\"}\n\n{\"id\":2,\"name\":\"project2\"}\n\n{\"id\":3,\"name\":\"project-internal-dev\",\"namespace\":{\"name\":\"🦊\"}}\n\n```\n\n\n### How to filter data structures with `jq`\n\n\nFilter items by passing `| select (...)` to `jq`. The filter takes a lambda\ncallback function as a comparator condition. When the item matches the\ncondition, it is returned to the caller.\n\n\nUse the dot indexer `.` to access dictionary keys and their values. Try to\nfilter for all items where the name is `project2`:\n\n\n```shell\n\n$ cat result.json | jq -c '.[] | select (.name == \"project2\")'\n\n{\"id\":2,\"name\":\"project2\"}\n\n```\n\n\nPractice this example by selecting the `id` with the value `2` instead of\nthe `name`.\n\n\n#### Filter with matching a string\n\n\nDuring tests, you may need to match different patterns instead of knowing\nthe full name. Think of projects that match a specific path or are located\nin a group where you only know the prefix. Simple string matches can be\nachieved with the `| contains (...)` function. It allows you to check\nwhether the given string is inside the target string – which requires the\nselected attribute to be of the string type.\n\n\nFor a filter with the select chain, the comparison condition needs to be\nchanged from the equal operator `==` to checking the attribute `.name` with\n`| contains (\"dev\")`.\n\n\n```shell\n\n$ cat result.json | jq -c '.[] | select (.name | contains (\"dev\") )'\n\n{\"id\":3,\"name\":\"project-internal-dev\",\"namespace\":{\"name\":\"🦊\"}}\n\n```\n\n\nSimple matches can be achieved with the `contains` function.\n\n\n#### Filter with matching regular expressions\n\n\nFor advanced string pattern matching, it is recommended to use regular\nexpressions. `jq` provides the [test function for this use\ncase](https://stedolan.github.io/jq/manual/#RegularexpressionsPCRE). Try to\nfilter for all projects which end with a number, represented by `\\d+`. Note\nthat the backslash `\\` needs to be escaped as `\\\\` for shell execution. `^`\ntests for beginning of the string, `$` is the ending check.\n\n\n```shell\n\n$ cat result.json | jq -c '.[] | select (.name | test (\"^project\\\\d+$\") )'\n\n{\"id\":1,\"name\":\"project1\"}\n\n{\"id\":2,\"name\":\"project2\"}\n\n```\n\n\nTip: You can [test and build the regular expression with\nregex101](https://regex101.com/) before test-driving it with `jq`.\n\n\n#### Access nested values\n\n\nKey value pairs in a dictionary may have a dictionary or array as a value.\n`jq` filters need to take this factor into account when filtering or\ntransforming the result. The example data structure provides\n`project-internal-dev` which has the key `namespace` and a value of a\ndictionary type.\n\n\n```shell\n  {\n    \"id\": 3,\n    \"name\": \"project-internal-dev\",\n    \"namespace\": {\n      \"name\": \"🦊\"\n    }\n  }\n```\n\n\n`jq` allows the user to specify the [array and dictionary\ntypes](https://stedolan.github.io/jq/manual/#TypesandValues) as `[]` and\n`{}` to be used in select chains with greater and less than comparisons. The\n`[]` brackets select filters for non-empty dictionaries for the `namespace`\nattribute, while the `{}` brackets select for all `null` (raw JSON) values.\n\n\n```shell\n\n$ cat result.json | jq -c '.[] | select (.namespace >={} )'\n\n{\"id\":3,\"name\":\"project-internal-dev\",\"namespace\":{\"name\":\"🦊\"}}\n\n\n$ cat result.json | jq -c '.[] | select (.namespace \u003C={} )'\n\n{\"id\":1,\"name\":\"project1\"}\n\n{\"id\":2,\"name\":\"project2\"}\n\n```\n\n\nThese methods can be used to access the name attribute of the namespace, but\nonly if the namespace contains values. Tip: You can chain multiple `jq`\ncalls by piping the result into another `jq` call. `.name` is a subkey of\nthe primary `.namespace` key.\n\n\n```shell\n\n$ cat result.json | jq -c '.[] | select (.namespace >={} )' | jq -c\n'.namespace.name'\n\n\"🦊\"\n\n```\n\n\nThe additional select command with non-empty namespaces ensures that only\ninitialized values for `.namespace.name` are returned. This is a safety\ncheck, and avoids receiving `null` values in the result you would need to\nfilter again.\n\n\n```shell\n\n$ cat result.json| jq -c '.[]' | jq -c '.namespace.name'\n\nnull\n\nnull\n\n\"🦊\"\n\n```\n\n\nBy using the additional check with `| select (.namespace >={} )`, you only\nget the expected results and do not have to filter empty `null` values.\n\n\n### How to expand the GitLab endpoint response\n\n\nSave the result from the API projects call and retry the examples above with\n`jq`.\n\n\n```shell\n\n$ curl \"https://gitlab.com/api/v4/projects\" -o result.json 2&>1 >/dev/null\n\n```\n\n\n### Validate CI/CD YAML with `jq` for Git hooks\n\n\nWhile writing this blog post, I learned that you can [escape and encode YAML\ninto JSON with\n`jq`](https://docs.gitlab.com/ee/api/lint.html#escape-yaml-for-json-encoding).\nThis trick comes in handy when automating YAML linting on the CLI, for\nexample as a Git pre-commit hook.\n\n\nLet’s take a look at the simplest way to test GitLab CI/CD from our\n[community meetup\nworkshops](https://gitlab.com/gitlab-de/swiss-meetup-2021-jan#resources). A\ncommon mistake with the first steps of the process can be missing the two\nspaces indent or missing whitespace between the dash and following command.\nThe following examples use `.gitlab-ci.error.yml` as a filename to showcase\nerrors and `.gitlab-ci.main.yml` for working examples.\n\n\n```shell\n\n$ vim .gitlab-ci.error.yml\n\n\nimage: alpine:latest\n\n\ntest:\n\nscript:\n  -exit 1\n```\n\n\nCommitting the change and waiting for the CI/CD pipeline to validate at\nruntime can be time-consuming. The [GitLab API provides a resource endpoint\n/ci/lint](https://docs.gitlab.com/ee/api/lint.html#validate-the-ci-yaml-configuration).\nA POST request with JSON-encoded YAML content will return a linting result\nfaster.\n\n\n#### Parse CI/CD YAML into JSON with jq\n\n\nYou can use jq to parse the raw YAML string into JSON:\n\n\n```shell\n\n$ jq --raw-input --slurp \u003C .gitlab-ci.error.yml\n\n\"image: alpine:latest\\n\\ntest:\\nscript:\\n  -exit 1\\n\"\n\n```\n\n\nThe `/ci/lint` API endpoint requires a JSON dictionary with `content` as\nkey, and the raw YAML string as a value. You can use `jq` to format the\ninput by using the arg parser:\n\n\n```shell\n\n§ jq --null-input --arg yaml \"$(\u003C.gitlab-ci.error.yml)\" '.content=$yaml'\n\n{\n  \"content\": \"image: alpine:latest\\n\\ntest:\\nscript:\\n  -exit 1\"\n}\n\n```\n\n\n#### Send POST request to /ci/lint\n\n\nThe next building block is to [send a POST request to the\n/ci/lint](https://docs.gitlab.com/ee/api/lint.html#validate-the-ci-yaml-configuration).\nThe request needs to specify the `Content-Type` header for the body. With\nusing the pipe `|` character, the JSON-encoded YAML configuration is fed\ninto the curl command call.\n\n\n```shell\n\n$ jq --null-input --arg yaml \"$(\u003C.gitlab-ci.error.yml)\" '.content=$yaml' \\\n\n| curl \"https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true\" \\\n\n--header 'Content-Type: application/json' --data @-\n\n{\"status\":\"invalid\",\"errors\":[\"jobs test config should implement a script:\nor a trigger: keyword\",\"jobs script config should implement a script: or a\ntrigger: keyword\",\"jobs config should contain at least one visible\njob\"],\"warnings\":[],\"merged_yaml\":\"\n",[787,789,9],{"slug":1711,"featured":6,"template":700},"devops-workflows-json-format-jq-ci-cd-lint","content:en-us:blog:devops-workflows-json-format-jq-ci-cd-lint.yml","Devops Workflows Json Format Jq Ci Cd Lint","en-us/blog/devops-workflows-json-format-jq-ci-cd-lint.yml","en-us/blog/devops-workflows-json-format-jq-ci-cd-lint",{"_path":1717,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1718,"content":1724,"config":1729,"_id":1731,"_type":13,"title":1732,"_source":15,"_file":1733,"_stem":1734,"_extension":18},"/en-us/blog/efficient-devsecops-workflows-hands-on-python-gitlab-api-automation",{"ogTitle":1719,"schema":1720,"ogImage":1721,"ogDescription":1722,"ogSiteName":685,"noIndex":6,"ogType":686,"ogUrl":1723,"title":1719,"canonicalUrls":1723,"description":1722},"Efficient DevSecOps workflows: Hands-on python-gitlab API automation","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Efficient DevSecOps workflows: Hands-on python-gitlab API automation\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2023-02-01\",\n      }","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659883/Blog/Hero%20Images/post-cover-image.jpg","The python-gitlab library is a useful abstraction layer for the GitLab API. Dive into hands-on examples and best practices in this tutorial.","https://about.gitlab.com/blog/efficient-devsecops-workflows-hands-on-python-gitlab-api-automation",{"title":1719,"description":1722,"authors":1725,"heroImage":1721,"date":1726,"body":1727,"category":785,"tags":1728},[1646],"2023-02-01","A friend once said in a conference presentation, “Manual work is a bug.\"\nWhen there are repetitive tasks in workflows, I tend to [come back to this\nquote](https://twitter.com/dnsmichi/status/1574087419237916672), and try to\nautomate as much as possible. For example, by querying a REST API to do an\ninventory of settings, or calling API actions to create new comments in\nGitLab issues/merge requests. The interaction with the GitLab REST API can\nbe done in different ways, using HTTP requests with curl (or\n[hurl](/blog/how-to-continously-test-web-apps-apis-with-hurl-and-gitlab-ci-cd/))\non the command line, or by writing a script in a programming language. The\nlatter can become reinventing the wheel again with raw HTTP requests code,\nand parsing the JSON responses.\n\n\nThanks to the wider GitLab community, many different languages are supported by API abstraction libraries. They provide support for all API attributes, add helper functions to get/create/delete objects, and generally aim to help developers focus. The [python-gitlab library](https://python-gitlab.readthedocs.io/en/stable/) is a feature-rich and easy-to-use library written in Python.\n\n\nIn this blog post, you will learn about the basic usage of the library by working with API objects, attributes, pagination and resultsets, and dive into more concrete use cases collecting data, printing summaries and writing data to the API to create comments and commits. There is a whole lot more to learn, with many of the use cases inspired by wider community questions on the forum, Hacker News, issues, etc.\n\n\nThis blog post is a long read, so feel free to stick with the beginner's tutorial or skip to the advanced [DevSecOps](https://about.gitlab.com/topics/devsecops/) use cases, development tips and code optimizations by navigating the table of contents:\n\n\n- [Getting started](#getting-started)\n\n- [Configuration](#configuration)\n\n- [Managing objects: The GitLab Object](#managing-objects-the-gitlab-object)\n    - [Objects managers and loading](#objects-managers-and-loading)\n    - [Pagination of results](#pagination-of-results)\n    - [Working with object relationships](#working-with-object-relationships)\n    - [Working with different object collection scopes](#working-with-different-object-collection-scopes)\n- [DevSecOps use cases for API read actions](#devsecops-use-cases-for-api-read-actions)\n    - [List branches by merged state](#list-branches-by-merged-state)\n    - [Print project settings for review: MR approval rules](#print-project-settings-for-review-mr-approval-rules)\n    - [Inventory: Get all CI/CD variables that are protected or masked](#inventory-get-all-cicd-variables-that-are-protected-or-masked)\n    - [Download a file from the repository](#download-a-file-from-the-repository)\n    - [Migration help: List all certificate-based Kubernetes clusters](#migration-help-list-all-certificate-based-kubernetes-clusters)\n    - [Team efficiency: Check if existing merge requests need to be rebased after merging a huge refactoring MR](#team-efficiency-check-if-existing-merge-requests-need-to-be-rebased-after-merging-a-huge-refactoring-mr)\n- [DevSecOps use cases for API write actions](#devsecops-use-cases-for-api-write-actions)\n    - [Move epics between groups](#move-epics-between-groups)\n    - [Compliance: Ensure that project settings are not overridden](#compliance-ensure-that-project-settings-are-not-overridden)\n    - [Taking notes, generate due date overview](#taking-notes-generate-due-date-overview)\n    - [Create issue index in a Markdown file, grouped by labels](#create-issue-index-in-a-markdown-file-grouped-by-labels)\n- [Advanced DevSecOps workflows](#advanced-devsecops-workflows)\n    - [Container images to run API scripts](#container-images-to-run-api-scripts)\n    - [CI/CD integration: Release and changelog generation](#cicd-integration-release-and-changelog-generation)\n    - [CI/CD integration: Pipeline report summaries](#cicd-integration-pipeline-report-summaries)\n- [Development tips](#development-tips)\n    - [Advanced custom configuration](#advanced-custom-configuration)\n    - [CI/CD code linting for different Python versions](#cicd-code-linting-for-different-python-versions)\n- [Optimize code and performance](#optimize-code-and-performance)\n    - [Lazy objects](#lazy-objects)\n    - [Object-oriented programming](#object-oriented-programming)\n- [More use cases](#more-use-cases)\n\n- [Conclusion](#conclusion)\n\n\n## Getting started\n\n\nThe python-gitlab documentation is a great resource for [getting started guides](https://python-gitlab.readthedocs.io/en/stable/api-usage.html), object types and their available methods, and combined workflow examples. Together with the [GitLab API resources documentation](https://docs.gitlab.com/ee/api/api_resources.html), which provides the object attributes that can be used, these are the best resources to get going.\n\n\nThe code examples in this blog post require Python 3.8+, and the `python-gitlab` library. Additional requirements are specified in the `requirements.txt` file – one example requires `pyyaml` for YAML config parsing. To follow and practice the use cases code, it is recommended to clone the project, install the requirements and run the scripts. Example with Homebrew on macOS:\n\n\n```shell\n\ngit clone https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python.git\n\n\ncd gitlab-api-python\n\n\nbrew install python\n\n\npip3 install -r requirements.txt\n\n\npython3 \u003Cscriptname>.py\n\n```\n\n\nThe scripts are intentionally not using a common shared library that provides generic functions for parameter reads, or additional helper functionality, for example. The idea is to show easy-to-follow examples that can be used stand-alone for testing, and only require installing the `python-gitlab` library as a dependency. Improving the code for production use is recommended. This can also help with building a maintained API tooling project that, for example, includes container images and CI/CD templates for developers to consume on a DevSecOps platform.\n\n\n## Configuration\n\n\nWithout configuration, python-gitlab will run unauthenticated requests against the default server `https://gitlab.com`. The most common configuration settings relate to the GitLab instance to connect to, and the authentication method by specifying access tokens. Python-gitlab supports different types of configuration: A configuration file or environment variables.\n\n\nThe [configuration file](https://python-gitlab.readthedocs.io/en/stable/cli-usage.html#cli-configuration) is available for the API library bindings, and the CLI (the CLI is not explained in this blog post). The configuration file supports [credential helpers](https://python-gitlab.readthedocs.io/en/stable/cli-usage.html#credential-helpers) to access tokens directly.\n\n\nEnvironment variables as an alternative configuration method provide an easy way to run the script on terminal, integrate into container images, and prepare them for running in CI/CD pipelines.\n\n\nThe configuration needs to be loaded into the Python script context. Start by importing the `os` library to fetch environment variables using the `os.environ.get()` method. The first parameter specifies the key, the second parameter sets the default value when the variable is not available in the environment.\n\n\n```python\n\nimport os\n\n\ngl_server = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\n\nprint(gl_server)\n\n```\n\n\nThe parametrization on the terminal can happen directly for the command only, or exported into the shell environment.\n\n\n```shell\n\n$ GL_SERVER=’https://gitlab.company.com’ python3 script.py\n\n\n$ export GL_SERVER=’https://gitlab.company.com’\n\n$ python3 script.py\n\n```\n\n\nIt is recommended to add safety checks to ensure that all variables are set before continuing to run the program. The following snippet imports the required libraries, reads the `GL_SERVER` environment variable and expects the user to set the `GL_TOKEN` variable. If not, the script prints and throws errors, and calls `sys.exit(1)` indicating an error status.\n\n\n```python\n\nimport gitlab\n\nimport os\n\nimport sys\n\n\nGITLAB_SERVER = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\nGITLAB_TOKEN = os.environ.get('GL_TOKEN')\n\n\nif not GITLAB_TOKEN:\n    print(\"Please set the GL_TOKEN env variable.\")\n    sys.exit(1)\n```\n\n\nWe will look into a more detailed example now which creates a connection to the API and makes an actual data request.\n\n\n## Managing objects: The GitLab object\n\n\nAny interaction with the API requires the GitLab object to be instantiated. This is the entry point to configure the GitLab server to connect, authenticate using access tokens, and more global settings for pagination, object loading and more.\n\n\nThe following example runs an unauthenticated request against GitLab.com. It is possible to access public API endpoints and for example get a specific [.gitignore template for Python](https://python-gitlab.readthedocs.io/en/stable/gl_objects/templates.html#gitignore-templates).\n\n\n[python_gitlab_object_unauthenticated.py](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_object_unauthenticated.py)\n\n\n```python\n\nimport gitlab\n\n\ngl = gitlab.Gitlab()\n\n\n# Get .gitignore templates without authentication\n\ngitignore_templates = gl.gitignores.get('Python')\n\n\nprint(gitignore_templates.content)\n\n```\n\n\nThe next sections provide more insights into:\n\n\n- [Objects managers and loading](#objects-managers-and-loading)\n\n- [Pagination of results](#pagination-of-results)\n\n- [Working with object relationships](#working-with-object-relationships)\n\n- [Working with different object collection scopes](#working-with-different-object-collection-scopes)\n\n\n### Objects managers and loading\n\n\nThe python-gitlab library provides access to GitLab resources using so-called “[managers](https://python-gitlab.readthedocs.io/en/stable/api-usage.html#managers)\". Each manager type implements methods to work with the datasets (list, get, etc.).\n\n\nThe script shows how to access subgroups, direct projects, all projects including subgroups, issues, epics and todos. These methods and API endpoint require authentication to access all attributes. The code snippet, therefore, uses variables to get the authentication token, and also uses the `GROUP_ID` variable to specify a main group at which to start searching.\n\n\n```python\n\n#!/usr/bin/env python\n\n\nimport gitlab\n\nimport os\n\nimport sys\n\n\nGITLAB_SERVER = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\n# https://gitlab.com/gitlab-de/use-cases/\n\nGROUP_ID = os.environ.get('GL_GROUP_ID', 16058698)\n\nGITLAB_TOKEN = os.environ.get('GL_TOKEN')\n\n\nif not GITLAB_TOKEN:\n    print(\"Please set the GL_TOKEN env variable.\")\n    sys.exit(1)\n\ngl = gitlab.Gitlab(GITLAB_SERVER, private_token=GITLAB_TOKEN)\n\n\n# Main\n\nmain_group = gl.groups.get(GROUP_ID)\n\n\nprint(\"Sub groups\")\n\nfor sg in main_group.subgroups.list():\n    print(\"Subgroup name: {sg}\".format(sg=sg.name))\n\nprint(\"Projects (direct)\")\n\nfor p in main_group.projects.list():\n    print(\"Project name: {p}\".format(p=p.name))\n\nprint(\"Projects (including subgroups)\")\n\nfor p in main_group.projects.list(include_subgroups=True, all=True):\n     print(\"Project name: {p}\".format(p=p.name))\n\nprint(\"Issues\")\n\nfor i in main_group.issues.list(state='opened'):\n    print(\"Issue title: {t}\".format(t=i.title))\n\nprint(\"Epics\")\n\nfor e in main_group.issues.list():\n    print(\"Epic title: {t}\".format(t=e.title))\n\nprint(\"Todos\")\n\nfor t in gl.todos.list(state='pending'):\n    print(\"Todo: {t} url: {u}\".format(t=t.body, u=t.target_url\n```\n\n\nYou can run the script [`python_gitlab_object_manager_methods.py`](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_object_manager_methods.py) by overriding the `GROUP_ID` variable on GitLab.com SaaS for your own group to analyze. The `GL_SERVER` variable needs to be specified for self-managed instance targets. `GL_TOKEN` must provide the personal access token.\n\n\n```shell\n\nexport GL_TOKEN=xxx\n\n\nexport GL_SERVER=”https://gitlab.company.com”\n\n\nexport GL_SERVER=”https://gitlab.com”\n\n\nexport GL_GROUP_ID=1234\n\n\npython3 python_gitlab_object_manager_methods.py\n\n```\n\n\nGoing forward, the example snippets won’t show the Python headers and environment variable parsing to focus on the algorithm and functionality. All scripts are open source under the MIT license and available in [this project](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python).\n\n\n### Pagination of results\n\n\nBy default, the GitLab API does not return all result sets and requires the clients to use [pagination](https://docs.gitlab.com/ee/api/rest/index.html#pagination) to iterate through all result pages. The python-gitlab library [allows users to specify the settings](https://python-gitlab.readthedocs.io/en/stable/api-usage.html#pagination) globally in the GitLab object, or on each `list()` call. By default, all result sets would fire API requests, which can slow down the script execution. The recommended way is using `iterator=True` which returns a generator object, and API calls are fired on-demand when accessing the object.\n\n\nThe following example searches for the group name `everyonecancontribute`, and uses keyset pagination with 100 results on each page. The iterator is set to true on `gl.groups.list(iterator=True)` to fetch new result sets on demand. If the searched group name is found, the loop breaks and prints a summary, including measuring the duration of the complete search request.\n\n\n```python\n\nSEARCH_GROUP_NAME=\"everyonecancontribute\"\n\n\n# Use keyset pagination\n\n# https://python-gitlab.readthedocs.io/en/stable/api-usage.html#pagination\n\ngl = gitlab.Gitlab(GITLAB_SERVER, private_token=GITLAB_TOKEN,\n    pagination=\"keyset\", order_by=\"id\", per_page=100)\n\n# Iterate over the list, and fire new API calls in case the result set does not match yet\n\ngroups = gl.groups.list(iterator=True)\n\n\nfound_page = 0\n\nstart = timer()\n\n\nfor group in groups:\n    if SEARCH_GROUP_NAME == group.name:\n        # print(group) # debug\n        found_page = groups.current_page\n        break\n\nend = timer()\n\n\nduration = f'{end-start:.2f}'\n\n\nif found_page > 0:\n    print(\"Pagination API example for Python with GitLab{desc} - found group {g} on page {p}, duration {d}s\".format(\n        desc=\", the DevSecOps platform\", g=SEARCH_GROUP_NAME, p=found_page, d=duration))\nelse:\n    print(\"Could not find group name '{g}', duration {d}\".format(g=SEARCH_GROUP_NAME, d=duration))\n```\n\n\nExecuting `python_gitlab_pagination.py` found the [everyonecancontribute group](https://gitlab.com/everyonecancontribute) on page 5.\n\n\n```shell\n\n$ python3 python_gitlab_pagination.py\n\nPagination API example for Python with GitLab, the DevSecOps platform - found group everyonecancontribute on page 5, duration 8.51s\n\n```\n\n\n### Working with object relationships\n\n\nWhen working with object relationships – for example, collecting all projects in a given group – additional steps need to be taken. The returned project objects provide limited attributes by default. Manageable objects require an additional `get()` call which requests the full project object from the API in the background. This on-demand workflow helps to avoid waiting times and traffic by reducing the immediately returned attributes.\n\n\nThe following example illustrates the problem by looping through all projects in a group, and tries to call the `project.branches.list()` function, raising an exception in the try/except flow. The second example gets a manageable project object and tries the function call again.\n\n\n```python\n\n# Main\n\ngroup = gl.groups.get(GROUP_ID)\n\n\n# Collect all projects in group and subgroups\n\nprojects = group.projects.list(include_subgroups=True, all=True)\n\n\nfor project in projects:\n    # Try running a method on a weak object\n    try:\n       print(\"🤔 Project: {pn} 💡 Branches: {b}\\n\".format(\n        pn=project.name,\n        b=\", \".join([x.name for x in project.branches.list()])))\n    except Exception as e:\n        print(\"Got exception: {e} \\n ===================================== \\n\".format(e=e))\n\n    # Retrieve a full manageable project object\n    # https://python-gitlab.readthedocs.io/en/stable/gl_objects/groups.html#examples\n    manageable_project = gl.projects.get(project.id)\n\n    # Print a method available on a manageable object\n    print(\"🤔 Project: {pn} 💡 Branches: {b}\\n\".format(\n        pn=manageable_project.name,\n        b=\", \".join([x.name for x in manageable_project.branches.list()])))\n```\n\n\nThe exception handler in the python-gitlab library prints the error message, and also links to the documentation. It is helpful to take a debugging note that objects might not be available to manage whenever you cannot access object attributes or function calls.\n\n\n```shell\n\n$ python3 python_gitlab_manageable_objects.py\n\n\n🤔 Project: GitLab API Playground 💡 Branches: cicd-demo-automated-comments, docs-mr-approval-settings, main\n\n\nGot exception: 'GroupProject' object has no attribute 'branches'\n\n\n\u003Cclass 'gitlab.v4.objects.projects.GroupProject'> was created via a\n\nlist() call and only a subset of the data may be present. To ensure\n\nall data is present get the object using a get(object.id) call. For\n\nmore details, see:\n\n\nhttps://python-gitlab.readthedocs.io/en/v3.8.1/faq.html#attribute-error-list\n =====================================\n```\n\n\nThe full script is located [here](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_manageable_objects.py).\n\n\n### Working with different object collection scopes\n\n\nSometimes, the script needs to collect all projects from a self-managed instance, or from a group with subgroups, or from a single project. The latter is helpful for faster testing on the required attributes, and the group fetch helps with testing at scale later. The following snippet collects all project objects into the `projects` list, and appends objects from different incoming configuration. You will also see the manageable object pattern for project in groups again.\n\n\n```python\n    # Collect all projects, or prefer projects from a group id, or a project id\n    projects = []\n\n    # Direct project ID\n    if PROJECT_ID:\n        projects.append(gl.projects.get(PROJECT_ID))\n\n    # Groups and projects inside\n    elif GROUP_ID:\n        group = gl.groups.get(GROUP_ID)\n\n        for project in group.projects.list(include_subgroups=True, all=True):\n            # https://python-gitlab.readthedocs.io/en/stable/gl_objects/groups.html#examples\n            manageable_project = gl.projects.get(project.id)\n            projects.append(manageable_project)\n\n    # All projects on the instance (may take a while to process)\n    else:\n        projects = gl.projects.list(get_all=True)\n```\n\n\nThe full example is located in [this script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/get_mr_approval_rules.py) for listing MR approval rules settings for specified project targets.\n\n\n## DevSecOps use cases for API read actions\n\n\nThe authenticated access token needs [`read_api` scope](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#personal-access-token-scopes).\n\n\nThe following use cases are discussed:\n\n\n- [List branches by merged state](#list-branches-by-merged-state)\n\n- [Print project settings for review: MR approval rules](#print-project-settings-for-review-mr-approval-rules)\n\n- [Inventory: Get all CI/CD variables that are protected or masked](#inventory-get-all-cicd-variables-that-are-protected-or-masked)\n\n- [Download a file from the repository](#download-a-file-from-the-repository)\n\n- [Migration help: List all certificate-based Kubernetes clusters](#migration-help-list-all-certificate-based-kubernetes-clusters)\n\n- [Team efficiency: Check if existing merge requests need to be rebased after merging a huge refactoring MR](#team-efficiency-check-if-existing-merge-requests-need-to-be-rebased-after-merging-a-huge-refactoring-mr)\n\n\n### List branches by merged state\n\n\nA common ask is to do some Git housekeeping in the project, and see how many merged and unmerged branches are floating around. [A question on the GitLab community forum](https://forum.gitlab.com/t/python-gitlab-project-branch-list-filter/80257) about filtering branch listings inspired me look into writing a [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/get_branches_by_state.py) that helps achieve this goal. The `branches.list()` method returns all branch objects that are stored in a temporary list for later processing for two loops: Collecting merged branch names, and not merged branch names. The `merged` attribute on the `branch` object is a boolean value indicating whether the branch has been merged.\n\n\n```python\n\nproject = gl.projects.get(PROJECT_ID, lazy=False, pagination=\"keyset\", order_by=\"updated_at\", per_page=100)\n\n\n# Get all branches\n\nreal_branches = []\n\nfor branch in project.branches.list():\n    real_branches.append(branch)\n\nprint(\"All branches\")\n\nfor rb in real_branches:\n    print(\"Branch: {b}\".format(b=rb.name))\n\n# Get all merged branches\n\nmerged_branches_names = []\n\nfor branch in real_branches:\n    if branch.default:\n        continue # ignore the default branch for merge status\n\n    if branch.merged:\n        merged_branches_names.append(branch.name)\n\nprint(\"Branches merged: {b}\".format(b=\", \".join(merged_branches_names)))\n\n\n# Get un-merged branches\n\nnot_merged_branches_names = []\n\nfor branch in real_branches:\n    if branch.default:\n        continue # ignore the default branch for merge status\n\n    if not branch.merged:\n        not_merged_branches_names.append(branch.name)\n\nprint(\"Branches not merged: {b}\".format(b=\", \".join(not_merged_branches_names)))\n\n```\n\n\nThe workflow is intentionally a step-by-step read, you can practice optimizing the Python code for the conditional branch name collection.\n\n\n\n### Print project settings for review: MR approval rules\n\n\nThe following [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/get_mr_approval_rules.py) walks through all collected project objects, and checks whether approval rules are specified. If the list length is greater than zero, it loops over the list and prints the settings using a JSON pretty-print method.\n\n\n```python\n    # Loop over projects and print the settings\n    # https://python-gitlab.readthedocs.io/en/stable/gl_objects/merge_request_approvals.html\n    for project in projects:\n        if len(project.approvalrules.list()) > 0:\n            #print(project) #debug\n            print(\"# Project: {name}, ID: {id}\\n\\n\".format(name=project.name_with_namespace, id=project.id))\n            print(\"[MR Approval settings]({url}/-/settings/merge_requests)\\n\\n\".format(url=project.web_url))\n\n            for ar in project.approvalrules.list():\n                print(\"## Approval rule: {name}, ID: {id}\".format(name=ar.name, id=ar.id))\n                print(\"\\n```json\\n\")\n                print(json.dumps(ar.attributes, indent=2)) # TODO: can be more beautiful, but serves its purpose with pretty print JSON\n                print(\"\\n```\\n\")\n\n```\n\n\n### Inventory: Get all CI/CD variables that are protected or masked\n\n\n[CI/CD variables](https://docs.gitlab.com/ee/ci/variables/) are helpful for pipeline parameterization, and can be configured globally on the instance, in groups and in projects. Secrets, passwords and otherwise sensitive information could be stored there, too. Sometimes it can be necessary to get an overview of all CI/CD variables that are either protected or masked to get a sense of how many variables need to be updated when rotating tokens for example.\n\n\nThe following [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/get_all_cicd_variables_masked_or_protected.py) gets all groups and projects and tries to collect the CI/CD variables from the global instance (requires admin permissions), groups and projects (requires maintainer/owner permissions). It prints all CI/CD variables that are either protected or masked, adding that a potential secret value is stored.\n\n\n```python\n\n#!/usr/bin/env python\n\n\nimport gitlab\n\nimport os\n\nimport sys\n\n\n# Helper function to evaluate secrets and print the variables\n\ndef eval_print_var(var):\n    if var.protected or var.masked:\n        print(\"🛡️🛡️🛡️ Potential secret: Variable '{name}', protected {p}, masked: {m}\".format(name=var.key,p=var.protected,m=var.masked))\n\nGITLAB_SERVER = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\nGITLAB_TOKEN = os.environ.get('GL_TOKEN') # token requires maintainer+ permissions. Instance variables require admin access.\n\nPROJECT_ID = os.environ.get('GL_PROJECT_ID') #optional\n\nGROUP_ID = os.environ.get('GL_GROUP_ID', 8034603) # https://gitlab.com/everyonecancontribute\n\n\nif not GITLAB_TOKEN:\n    print(\"🤔 Please set the GL_TOKEN env variable.\")\n    sys.exit(1)\n\ngl = gitlab.Gitlab(GITLAB_SERVER, private_token=GITLAB_TOKEN)\n\n\n# Collect all projects, or prefer projects from a group id, or a project id\n\nprojects = []\n\n# Collect all groups, or prefer group from a group id\n\ngroups = []\n\n\n# Direct project ID\n\nif PROJECT_ID:\n    projects.append(gl.projects.get(PROJECT_ID))\n\n# Groups and projects inside\n\nelif GROUP_ID:\n    group = gl.groups.get(GROUP_ID)\n\n    for project in group.projects.list(include_subgroups=True, all=True):\n        # https://python-gitlab.readthedocs.io/en/stable/gl_objects/groups.html#examples\n        manageable_project = gl.projects.get(project.id)\n        projects.append(manageable_project)\n\n    groups.append(group)\n\n# All projects/groups on the instance (may take a while to process, use iterators to fetch on-demand).\n\nelse:\n    projects = gl.projects.list(iterator=True)\n    groups = gl.groups.list(iterator=True)\n\nprint(\"# List of all CI/CD variables marked as secret (instance, groups, projects)\")\n\n\n# https://python-gitlab.readthedocs.io/en/stable/gl_objects/variables.html\n\n\n# Instance variables (if the token has permissions)\n\nprint(\"Instance variables, if accessible\")\n\ntry:\n    for i_var in gl.variables.list(iterator=True):\n        eval_print_var(i_var)\nexcept:\n    print(\"No permission to fetch global instance variables, continueing without.\")\n    print(\"\\n\")\n\n# group variables (maintainer permissions for groups required)\n\nfor group in groups:\n    print(\"Group {n}, URL: {u}\".format(n=group.full_path, u=group.web_url))\n    for g_var in group.variables.list(iterator=True):\n        eval_print_var(g_var)\n\n    print(\"\\n\")\n\n# Loop over projects and print the settings\n\nfor project in projects:\n    # skip archived projects, they throw 403 errors\n    if project.archived:\n        continue\n\n    print(\"Project {n}, URL: {u}\".format(n=project.path_with_namespace, u=project.web_url))\n    for p_var in project.variables.list(iterator=True):\n        eval_print_var(p_var)\n\n    print(\"\\n\")\n```\n\n\nThe script intentionally does not print the variable values, this is left as an exercise for safe environments. The recommended way of storing secrets is to [use external providers](https://docs.gitlab.com/ee/ci/secrets/).\n\n\n### Download a file from the repository\n\n\nThe [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/get_raw_file_content.py) goal is download a file path from a specified branch name, and store its content in a new file.\n\n\n```python\n\n# Goal: Try to download README.md from https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/README.md\n\nFILE_NAME = 'README.md'\n\nBRANCH_NAME = 'main'\n\n\n# Search the file in the repository tree and get the raw blob\n\nfor f in project.repository_tree():\n    print(\"File path '{name}' with id '{id}'\".format(name=f['name'], id=f['id']))\n\n    if f['name'] == FILE_NAME:\n        f_content = project.repository_raw_blob(f['id'])\n        print(f_content)\n\n# Alternative approach: Get the raw file from the main branch\n\nraw_content = project.files.raw(file_path=FILE_NAME, ref=BRANCH_NAME)\n\nprint(raw_content)\n\n\n# Store the file on disk\n\nwith open('raw_README.md', 'wb') as f:\n    project.files.raw(file_path=FILE_NAME, ref=BRANCH_NAME, streamed=True, action=f.write)\n```\n\n\n### Migration help: List all certificate-based Kubernetes clusters\n\n\nThe certificate-based integration of Kubernetes clusters into GitLab [was deprecated](https://docs.gitlab.com/ee/update/deprecations.html#self-managed-certificate-based-integration-with-kubernetes). To help with migration plans, the inventory of existing groups and projects can be automated using the GitLab API.\n\n\n\n```python\n\ngroups = [ ]\n\n\n# get GROUP_ID group\n\ngroups.append(gl.groups.get(GROUP_ID))\n\n\nfor group in groups:\n    for sg in group.subgroups.list(include_subgroups=True, all=True):\n        real_group = gl.groups.get(sg.id)\n        groups.append(real_group)\n\ngroup_clusters = {}\n\nproject_clusters = {}\n\n\nfor group in groups:\n    #Collect group clusters\n    g_clusters = group.clusters.list()\n\n    if len(g_clusters) > 0:\n        group_clusters[group.id] = g_clusters\n\n    # Collect all projects in group and subgroups and their clusters\n    projects = group.projects.list(include_subgroups=True, all=True)\n\n    for project in projects:\n        # https://python-gitlab.readthedocs.io/en/stable/gl_objects/groups.html#examples\n        manageable_project = gl.projects.get(project.id)\n\n        # skip archived projects\n        if project.archived:\n            continue\n\n        p_clusters = manageable_project.clusters.list()\n\n        if len(p_clusters) > 0:\n            project_clusters[project.id] = p_clusters\n\n# Print summary\n\nprint(\"## Group clusters\\n\\n\")\n\nfor g_id, g_clusters in group_clusters.items():\n    url = gl.groups.get(g_id).web_url\n    print(\"Group ID {g_id}: {u}\\n\\n\".format(g_id=g_id, u=url))\n    print_clusters(g_clusters)\n\nprint(\"## Project clusters\\n\\n\")\n\nfor p_id, p_clusters in project_clusters.items():\n    url = gl.projects.get(p_id).web_url\n    print(\"Project ID {p_id}: {u}\\n\\n\".format(p_id=p_id, u=url))\n    print_clusters(p_clusters)\n```\n\n\nThe full script is available [here](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/list_cert_based_kubernetes_clusters.py).\n\n\n### Team efficiency: Check if existing merge requests need to be rebased after merging a huge refactoring MR\n\n\nThe [GitLab handbook](https://handbook.gitlab.com/handbook/) repository is a large monorepo with many merge requests created, reviewed, approved and merged. Some reviews take longer than others, and some merge requests touch multiple pages when renaming a string, or [all handbook pages](/handbook/about/#count-handbook-pages). The marketing handbook needed restructuring (think of code refactoring), and as such, many directories and paths were moved or renamed. [The issue tasks](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/13991#tasks) grew over time, and I was worried that other merge requests would run into conflicts after merging the huge changes. I remembered that the python-gitlab can fetch all merge requests in a given project, including details on the Git branch, source paths changed and much more.\n\n\nThe resulting script configures a list of source paths that are touched by all merge requests, and checks against the merge request diff with `mr.diffs.list()` and comparing if a pattern matches against the value in `old_path`. If a match is found, the script logs it, and saves the merge request in the `seen_mr` dictionary for the summary later. There are additional attributes collected to allow printing a Markdown task list with URLs for easier copy-paste into [issue descriptions](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/13991#additional-tasks). The full script is located [here](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/search_mr_contains_updated_path.py).\n\n\n\n```python\n\nPATH_PATTERNS = [\n    'path/to/handbook/source/page.md',\n]\n\n\n# Only list opened MRs\n\n# https://python-gitlab.readthedocs.io/en/stable/gl_objects/merge_requests.html#project-merge-requests\n\nmrs = project.mergerequests.list(state='opened', iterator=True)\n\n\nseen_mr = {}\n\n\nfor mr in mrs:\n    # https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-request-diffs\n    real_mr = project.mergerequests.get(mr.get_id())\n    real_mr_id = real_mr.attributes['iid']\n    real_mr_url = real_mr.attributes['web_url']\n\n    for diff in real_mr.diffs.list(iterator=True):\n        real_diff = real_mr.diffs.get(diff.id)\n\n        for d in real_diff.attributes['diffs']:\n            for p in PATH_PATTERNS:\n                if p in d['old_path']:\n                    print(\"MATCH: {p} in MR {mr_id}, status '{s}', title '{t}' - URL: {mr_url}\".format(\n                        p=p,\n                        mr_id=real_mr_id,\n                        s=mr_status,\n                        t=real_mr.attributes['title'],\n                        mr_url=real_mr_url))\n\n                    if not real_mr_id in seen_mr:\n                        seen_mr[real_mr_id] = real_mr\n\nprint(\"\\n# MRs to update\\n\")\n\n\nfor id, real_mr in seen_mr.items():\n    print(\"- [ ] !{mr_id} - {mr_url}+ Status: {s}, Title: {t}\".format(\n        mr_id=id,\n        mr_url=real_mr.attributes['web_url'],\n        s=real_mr.attributes['detailed_merge_status'],\n        t=real_mr.attributes['title']))\n```\n\n\n\n## DevSecOps use cases for API write actions\n\n\nThe authenticated access token needs full [`api` scope](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#personal-access-token-scopes).\n\n\nThe following use cases are discussed:\n\n\n- [Move epics between groups](#move-epics-between-groups)\n\n- [Compliance: Ensure that project settings are not overridden](#compliance-ensure-that-project-settings-are-not-overridden)\n\n- [Taking notes, generate due date overview](#taking-notes-generate-due-date-overview)\n\n- [Create issue index in a Markdown file, grouped by labels](#create-issue-index-in-a-markdown-file-grouped-by-labels)\n\n\n### Move epics between groups\n\n\nSometimes it is necessary to move epics, similar to issues, into a different group. A question in the GitLab marketing Slack channel inspired me to look into a [feature proposal for the UI](https://gitlab.com/gitlab-org/gitlab/-/issues/12689), [quick actions](/blog/improve-your-gitlab-productivity-with-these-10-tips/), and later, thinking about writing an API script to automate the steps. The idea is simple: Move an epic from a source group to a target group, and copy its title, description and labels. Since epics allow to group issues, they need to be reassigned to the target epic, too. Parent-child epic relationships need to be taken into account to: All child epics of the source epics need to be reassigned to the target epic.\n\n\nThe following script looks up all source [epic attributes](https://python-gitlab.readthedocs.io/en/stable/gl_objects/epics.html) first, and then creates a new target epic with minimal attributes: title and description. The labels list is copied and the changes are persisted with the `save()` call. The issues assigned to the epic need to be re-created in the target epic. The `create()` call actually creates the relationship item, not a new issue object itself. The child epics move requires a different approach, since the relationship is vice versa: The `parent_id` on the child epic needs to be compared against the source epic ID, and if matching, updated to the target epic ID. After copying everything successfully, the source epic needs to be changed into the `closed` state.\n\n\n\n```python\n\n#!/usr/bin/env python\n\n\n# Description: Show how epics can be moved between groups, including title, description, labels, child epics and issues.\n\n# Requirements: python-gitlab Python libraries. GitLab API write access, and maintainer access to all configured groups/projects.\n\n# Author: Michael Friedrich \u003Cmfriedrich@gitlab.com>\n\n# License: MIT, (c) 2023-present GitLab B.V.\n\n\nimport gitlab\n\nimport os\n\nimport sys\n\n\nGITLAB_SERVER = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\n# https://gitlab.com/gitlab-de/use-cases/gitlab-api\n\nSOURCE_GROUP_ID = os.environ.get('GL_SOURCE_GROUP_ID', 62378643)\n\n# https://gitlab.com/gitlab-de/use-cases/gitlab-api/epic-move-target\n\nTARGET_GROUP_ID = os.environ.get('GL_TARGET_GROUP_ID', 62742177)\n\n# https://gitlab.com/groups/gitlab-de/use-cases/gitlab-api/-/epics/1\n\nEPIC_ID = os.environ.get('GL_EPIC_ID', 1)\n\nGITLAB_TOKEN = os.environ.get('GL_TOKEN')\n\n\nif not GITLAB_TOKEN:\n    print(\"Please set the GL_TOKEN env variable.\")\n    sys.exit(1)\n\ngl = gitlab.Gitlab(GITLAB_SERVER, private_token=GITLAB_TOKEN)\n\n\n# Main\n\n# Goal: Move epic to target group, including title, body, labels, and child epics and issues.\n\nsource_group = gl.groups.get(SOURCE_GROUP_ID)\n\ntarget_group = gl.groups.get(TARGET_GROUP_ID)\n\n\n# Create a new target epic and copy all its items, then close the source epic.\n\nsource_epic = source_group.epics.get(EPIC_ID)\n\n# print(source_epic) #debug\n\n\nepic_title = source_epic.title\n\nepic_description = source_epic.description\n\nepic_labels = source_epic.labels\n\nepic_issues = source_epic.issues.list()\n\n\n# Create the epic with minimal attributes\n\ntarget_epic = target_group.epics.create({\n    'title': epic_title,\n    'description': epic_description,\n})\n\n\n# Assign the list\n\ntarget_epic.labels = epic_labels\n\n\n# Persist the changes in the new epic\n\ntarget_epic.save()\n\n\n# Epic issues need to be re-assigned in a loop\n\nfor epic_issue in epic_issues:\n    ei = target_epic.issues.create({'issue_id': epic_issue.id})\n\n# Child epics need to update their parent_id to the new epic\n\n# Need to search in all epics, use lazy object loading\n\nfor sge in source_group.epics.list(lazy=True):\n    # this epic has the source epic as parent epic?\n    if sge.parent_id == source_epic.id:\n        # Update the parent id\n        sge.parent_id = target_epic.id\n        sge.save()\n\nprint(\"Copied source epic {source_id} ({source_url}) to target epic {target_id} ({target_url})\".format(\n    source_id=source_epic.id, source_url=source_epic.web_url,\n    target_id=target_epic.id, target_url=target_epic.web_url))\n\n# Close the old epic\n\nsource_epic.state_event = 'close'\n\nsource_epic.save()\n\nprint(\"Closed source epic {source_id} ({source_url})\".format(\n    source_id=source_epic.id, source_url=source_epic.web_url))\n\n```\n\n\n\n```shell\n\n$  python3 move_epic_between_groups.py\n\nCopied source epic 725341 (https://gitlab.com/groups/gitlab-de/use-cases/gitlab-api/-/epics/1) to target epic 725358 (https://gitlab.com/groups/gitlab-de/use-cases/gitlab-api/epic-move-target/-/epics/6)\n\nClosed source epic 725341 (https://gitlab.com/groups/gitlab-de/use-cases/gitlab-api/-/epics/1)\n\n```\n\n\n\nThe [target epic](https://gitlab.com/groups/gitlab-de/use-cases/gitlab-api/epic-move-target/-/epics/5) was created and shows the expected result: Same title, description, labels, child epic, and issues.\n\n\n![Target epic which has all attributes copied from the source epic: title, description, labels, child epics, issues](/images/blogimages/efficient-devsecops-workflows-python-gitlab-handson/python_gitlab_moved_epic_with_all_attributes.png){: .shadow}\n\n\n**Exercise**: The script does not copy [comments](https://python-gitlab.readthedocs.io/en/stable/gl_objects/notes.html) and [discussion threads](https://python-gitlab.readthedocs.io/en/stable/gl_objects/discussions.html) yet. Research and help update the script – merge requests welcome!\n\n\n\n### Compliance: Ensure that project settings are not overridden\n\n\nProject and group settings may be accidentally changed by team members with maintainer permissions. Compliance requirements need to be met. Another use case is to manage configuration with Infrastructure as Code tools, and ensure that GitLab instance/group/project/etc. configuration is persisted and always the same. Tools like Ansible or Terraform can invoke an API script, or use the python-gitlab library to perform tasks to manage settings.\n\n\nThe following example only has the `main` branch protected.\n\n\n![GitLab project settings for repositories and protected branches, main branch](/images/blogimages/efficient-devsecops-workflows-python-gitlab-handson/python_gitlab_protected_branches_settings_main.png){: .shadow}\n\n\nLet us assume that a new `production` branch has been added and should be protected, too. The following [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/enforce_protected_branches.py) defines the dictionary of protected branches and their access levels for push/merge permissions to maintainer level, and builds the comparison logic around the [python-gitlab protected branches documentation](https://python-gitlab.readthedocs.io/en/stable/gl_objects/protected_branches.html).\n\n\n\n```python\n\n#!/usr/bin/env python\n\n\nimport gitlab\n\nimport os\n\nimport sys\n\n\nGITLAB_SERVER = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\n# https://gitlab.com/gitlab-de/use-cases/\n\nGROUP_ID = os.environ.get('GL_GROUP_ID', 16058698)\n\nGITLAB_TOKEN = os.environ.get('GL_TOKEN')\n\n\nPROTECTED_BRANCHES = {\n    'main': {\n        'merge_access_level': gitlab.const.AccessLevel.MAINTAINER,\n        'push_access_level': gitlab.const.AccessLevel.MAINTAINER\n    },\n    'production': {\n        'merge_access_level': gitlab.const.AccessLevel.MAINTAINER,\n        'push_access_level': gitlab.const.AccessLevel.MAINTAINER\n    },\n}\n\n\nif not GITLAB_TOKEN:\n    print(\"Please set the GL_TOKEN env variable.\")\n    sys.exit(1)\n\ngl = gitlab.Gitlab(GITLAB_SERVER, private_token=GITLAB_TOKEN)\n\n\n# Main\n\ngroup = gl.groups.get(GROUP_ID)\n\n\n# Collect all projects in group and subgroups\n\nprojects = group.projects.list(include_subgroups=True, all=True)\n\n\nfor project in projects:\n    # Retrieve a full manageable project object\n    # https://python-gitlab.readthedocs.io/en/stable/gl_objects/groups.html#examples\n    manageable_project = gl.projects.get(project.id)\n\n    # https://python-gitlab.readthedocs.io/en/stable/gl_objects/protected_branches.html\n    protected_branch_names = []\n\n    for pb in manageable_project.protectedbranches.list():\n        manageable_protected_branch = manageable_project.protectedbranches.get(pb.name)\n        print(\"Protected branch name: {n}, merge_access_level: {mal}, push_access_level: {pal}\".format(\n            n=manageable_protected_branch.name,\n            mal=manageable_protected_branch.merge_access_levels,\n            pal=manageable_protected_branch.push_access_levels\n        ))\n\n        protected_branch_names.append(manageable_protected_branch.name)\n\n    for branch_to_protect, levels in PROTECTED_BRANCHES.items():\n        # Fix missing protected branches\n        if branch_to_protect not in protected_branch_names:\n            print(\"Adding branch {n} to protected branches settings\".format(n=branch_to_protect))\n            p_branch = manageable_project.protectedbranches.create({\n                'name': branch_to_protect,\n                'merge_access_level': gitlab.const.AccessLevel.MAINTAINER,\n                'push_access_level': gitlab.const.AccessLevel.MAINTAINER\n            })\n```\n\n\nRunning the script prints the existing `main` branch, and a note that `production` will be updated. The screenshot from the repository settings proves this action.\n\n\n```\n\n$ python3 enforce_protected_branches.py                                                ─╯\n\nProtected branch name: main, merge_access_level: [{'id': 67294702, 'access_level': 40, 'access_level_description': 'Maintainers', 'user_id': None, 'group_id': None}], push_access_level: [{'id': 68546039, 'access_level': 40, 'access_level_description': 'Maintainers', 'user_id': None, 'group_id': None}]\n\nAdding branch production to protected branches settings\n\n```\n\n\n![GitLab project settings for repositories and protected branches, main and production branch](/images/blogimages/efficient-devsecops-workflows-python-gitlab-handson/python_gitlab_protected_branches_settings_main_production.png){: .shadow}\n\n\n\n### Taking notes, generate due date overview\n\n\nA [Hacker News discussion about note-taking tools](https://news.ycombinator.com/item?id=32155848) inspired me to take a look into creating a Markdown table overview, fetched from files that take notes, and sorted by the parsed due date. The script is located [here](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/generate_snippets_index_by_due_date.py) and more complex to understand.\n\n\n```\n\n# 2022-07-19 Notes\n\n\nHN topic about taking notes: https://news.ycombinator.com/item?id=32152935\n\n\n\u003C!--\n\n---\n\nTags: DevOps, Learn\n\nDue: 2022-08-01\n\n---\n\n-->\n\n\n```\n\n\n### Create issue index in a Markdown file, grouped by labels\n\n\nA similar Hacker News question inspired me to write a [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/generate_issue_index_grouped_by_label.py) that parses all issues in a GitLab project by labels, and creates or updates a Markdown index file in the same repository. The issues are grouped by label.\n\n\nFirst, the issues are fetched from the project, including all labels, and stored in the `index` dictionary.\n\n\n```python\n\np = gl.projects.get(PROJECT_ID)\n\n\nlabels = p.labels.list()\n\n\nindex={}\n\n\nfor i in p.issues.list():\n    for l in i.labels:\n        if l not in index:\n            index[l] = []\n\n        index[l].append(\"#{id} - {title}\".format(id=i.id, title=i.title))\n```\n\n\nThe second step is to create a Markdown formatted listing based on the collected index data, with the label name as key, holding a list of issue strings.\n\n\n```python\n\nindex_str = \"\"\"# Issue Overview\n\n_Grouped by issue labels._\n\n\"\"\"\n\n\nfor l_name, i_list in index.items():\n    index_str += \"\\n## {label} \\n\\n\".format(label=l_name)\n\n    for i in i_list:\n        index_str += \"- {title}\\n\".format(title=i)\n```\n\n\nThe last step is to create a new file in the repository, or update an existing one. This is a little tricky because the API expects you to define the action and will throw an error if you try to update a nonexistent file. The first condition checks whether the file path exists in the repository, and then defines the `action` attribute. The `data` dictionary gets built, with the final `commits.create()` method called.\n\n\n```python\n\n# Dump index_str to FILE_NAME\n\n# Create as new commit\n\n# See https://docs.gitlab.com/ce/api/commits.html#create-a-commit-with-multiple-files-and-actions\n\n# for actions detail\n\n\n# Check if file exists, and define commit action\n\nf = p.files.get(file_path=FILE_NAME, ref=REF_NAME)\n\nif not f:\n    action='create'\nelse:\n    action='update'\n\ndata = {\n    'branch': REF_NAME,\n    'commit_message': 'Generate new index, {d}'.format(d=date.today()),\n    'actions': [\n        {\n            'action': action,\n            'file_path': FILE_NAME,\n            'content': index_str\n        }\n    ]\n}\n\n\ncommit = p.commits.create(data)\n\n```\n\n\n## Advanced DevSecOps workflows\n\n\n- [Container images to run API scripts](#container-images-to-run-api-scripts)\n\n- [CI/CD integration: Release and changelog generation](#cicd-integration-release-and-changelog-generation)\n\n- [CI/CD integration: Pipeline report summaries](#cicd-integration-pipeline-report-summaries)\n\n\n### Container images to run API scripts\n\n\nInstalling the Python interpreter and dependent libraries into the operating system may not always work, or it may be a barrier to using the API scripts. A container image that can be pulled from the GitLab registry is a good first step towards more DevSecOps automation and future CI/CD integrations, and provides a tested environment. The python-gitlab project [provides container images](https://python-gitlab.readthedocs.io/en/stable/index.html#using-the-docker-images) which can be used for testing.\n\n\nThe cloned script repository can be mounted into the container, and the settings are configured using environment variables. Example with Docker CLI:\n\n\n```shell\n\n$ docker run -ti -v \"`pwd`:/app\" \\\n  -e \"GL_SERVER=http://gitlab.com\" \\\n  -e \"GL_TOKEN=$GITLAB_TOKEN\" \\\n  -e \"GL_GROUP_ID=16058698\" \\\nregistry.gitlab.com/python-gitlab/python-gitlab:slim-bullseye \\\n\npython /app/python_gitlab_manageable_objects.py\n\n```\n\n\n### CI/CD integration: Release and changelog generation\n\n\nCreating a Git tag and a release in GitLab often requires a changelog attached. This provides a summary into all Git commits, all merged merge requests, or something similar that is easier to consume for everyone interested in the changes in this new release. Automating the changelog generation in CI/CD pipelines is possible using the GitLab API. The simplest list uses the Git commit history shown in the [`create_simple_changelog_from_git_history.py`](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/create_simple_changelog_from_git_history.py) script below:\n\n\n\n```python\n\nproject = gl.projects.get(PROJECT_ID)\n\ncommits = project.commits.list(ref_name='main', lazy=True, iterator=True)\n\n\nprint(\"# Changelog\")\n\n\nfor commit in commits:\n    # Generate a markdown formatted list with URLs\n    print(\"- [{text}]({url}) ({name})\".format(text=commit.title, url=commit.web_url, name=commit.author_name))\n```\n\n\nExecuting the script on the [o11y.love project](https://gitlab.com/everyonecancontribute/observability/o11y.love) will print a Markdown list with URLs.\n\n\n```shell\n\n$ python3 create_changelog_from_git_history.py\n\n# Changelog\n\n- [Merge branch 'topics-ebpf-opentelemetry' into 'main'](https://gitlab.com/everyonecancontribute/observability/o11y.love/-/commit/75df97e13e0f429803dc451aac7fee080a51f44c) (Michael Friedrich)\n\n- [Move eBPF/OpenTelemetry into dedicated topics pages ](https://gitlab.com/everyonecancontribute/observability/o11y.love/-/commit/8fa4233630ff8c1d65aff589bd31c4c2f5df36cb) (Michael Friedrich)\n\n- [Merge branch 'workshop-add-k8s-o11y-toc' into 'main'](https://gitlab.com/everyonecancontribute/observability/o11y.love/-/commit/8b7949b19af6aa6bf25f73ca1ffe8616a7dbaa00) (Michael Friedrich)\n\n- [Add TOC for Kubesimplify Kubernetes Observability workshop ](https://gitlab.com/everyonecancontribute/observability/o11y.love/-/commit/63c8ad587f43e3926e6749a62c33ad0b6f229f47) (Michael Friedrich)\n\n\n...\n\n```\n\n\n**Exercise**: The script is not production ready yet but should get you going to group by commits by Git tag/release, filter merge commits, attach the changelog file or content into the [GitLab release details](https://docs.gitlab.com/ee/api/releases/), etc.\n\n\n### CI/CD integration: Pipeline report summaries\n\n\nWhen developing new API script in Python, a CI/CD integration with automated runs can be desired, too. My recommendation is to focus on writing and testing the script stand-alone on the command line first, and once it works reliably, adapt the code to run the script to perform actions in CI/CD, too. After writing a few scripts, and practicing a lot, you will have learned to write code that can be executed on the CLI, in containers and in CI/CD jobs.\n\n\nA good preparation for CI/CD is to focus on environment variables to configure the script. The environment variables can be defined as CI/CD variables, and there is no extra work with additional configuration files, or command line parameters involved. This keeps the CI/CD configuration footprint small and reusable, too.\n\n\nAn example integration to automatically create security summaries as markdown comment in a merge request was described in the [\"Fantastic Infrastructure-as-Code security attacks and how to find them\" blog post](/blog/fantastic-infrastructure-as-code-security-attacks-and-how-to-find-them/#integrations-into-cicd-and-merge-requests-for-review). This use case required research and testing before actually writing the full API script:\n\n\n1. Read the python-gitlab documentation to learn how [merge request comments (notes)](https://python-gitlab.readthedocs.io/en/stable/gl_objects/notes.html#project-notes) can be created.\n\n2. Create a test project and a test merge request for testing.\n\n3. Start writing code which instantiates the GitLab connection object, fetches the project object, and gets the merge request object from a pre-defined ID.\n\n4. Run `mr.notes.create({‘body’: ‘This is a test by dnsmichi’})`\n\n5. Iterate on the body content and pre-fill a string with a markdown table.\n\n6. Fetch pre-defined CI/CD variables to get the `CI_MERGE_REQUEST_ID` value which will be required to update as target.\n\n6. Verify the API permissions and learn that the CI job token is not sufficient.\n\n7. Implement the full algorithm, integrated CI/CD testing and add documentation.\n\n\nThe script runs continuously after security scans have been completed with a report. Another use case can be using [Pipeline schedules](https://docs.gitlab.com/ee/ci/pipelines/schedules.html) which provide synchronization capabilities, and the comments get posted to an issue summary.\n\n\n## Development tips\n\n\nCode and abstraction libraries are helpful but sometimes it can be hard to see the problem why an attribute or object does not provide the expected behavior. It is helpful to take a step back, and look into different ways to fetch data from the REST API, for example [using jq and curl](/blog/devops-workflows-json-format-jq-ci-cd-lint/). The [GitLab CLI](/blog/introducing-the-gitlab-cli/) can also be used to query the API and get immediate results.\n\n\nDeveloping scripts that interact with APIs can become a repetitive task, adding more needed attributes, and the need to learn about object relations, methods and how to store the retrieved data. Especially for larger datasets, it can be a good idea to use the JSON library to dump data structures into a file cache on disk, and provide a debug configuration option to read the data from that file, instead of firing the API requests again all the time. This also helps to mitigate potential rate limiting.\n\n\nAdding timing points to the code can help measure the performance, and efficiency of the algorithm used. The following snippet [measures the duration](https://stackoverflow.com/questions/7370801/how-do-i-measure-elapsed-time-in-python ) of requests to retrieve the merge request status. It is part of a script that was used to analyze a potential problem with the `detailed_merge_status` attribute in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/386661#note_1237757295).\n\n\n```\n\nmrs = project.mergerequests.list(state='opened', iterator=True, with_merge_status_recheck=True)\n\n\nfor mr in mrs:\n    start = timer()\n    #print(mr.attributes) #debug\n    # https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-request-diffs\n    real_mr = project.mergerequests.get(mr.get_id())\n\n    print(\"- [ ] !{mr_id} - {mr_url}+ Status: {s}, Title: {t}\".format(\n        mr_id=real_mr.attributes['iid'],\n        mr_url=real_mr.attributes['web_url'],\n        s=real_mr.attributes['detailed_merge_status'],\n        t=real_mr.attributes['title']))\n\n    end = timer()\n    duration = end - start\n    if duration > 1.0:\n        print(\"ALERT: > 1s \")\n    print(\"> Execution time took {s}s\".format(s=(duration)))\n```\n\n\nMore tips are discussed in the following sections:\n\n\n- [Advanced custom configuration](#advanced-custom-configuration)\n\n- [CI/CD code linting for different Python versions](#cicd-code-linting-for-different-python-versions)\n\n\n### Advanced custom configuration\n\n\nWhen you are developing a script that requires advanced custom configuration, choose a format that fits best into existing infrastructure and development guidelines. Python provides libraries for parsing YAML, JSON, etc. The following example configuration file and script showcase a YAML configuration option. It is based on [a script that automatically updates a list of issues/epics](https://gitlab.com/gitlab-de/gitlab-api-automated-commenter) with a comment, reminding responsible team members for a recurring update for a cross-functional initiative at GitLab.\n\n\n[python_gitlab_custom_yaml_config.yml](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_custom_yaml_config.yml)\n\n```yaml\n\ntasks:\n  - name: \"Backend\"\n    url: \"https://gitlab.com/group1/project2/-/issues/1\"\n  - name: \"Frontend\"\n    url: \"https://gitlab.com/group2/project4/-/issues/2\"\n```\n\n\n[python_gitlab_custom_script_config_yaml.py](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_custom_script_config_yaml.py)\n\n```python\n\nimport os\n\nimport yaml\n\n\nCONFIG_FILE = os.environ.get('GL_CONFIG_FILE', \"python_gitlab_custom_yaml_config.yml\")\n\n\n# Read config\n\nwith open(CONFIG_FILE, mode=\"rt\", encoding=\"utf-8\") as file:\n    config = yaml.safe_load(file)\n    #print(config) #debug\n\ntasks = []\n\nif \"tasks\" in config:\n    tasks = config['tasks']\n\n# Process the tasks\n\nfor task in tasks:\n    print(\"Task name: '{n}' Issue URL to update: {id}\".format(n=task['name'], id=task['url']))\n    # print(task) #debug\n```\n\n\n```shell\n\n$ python3 python_gitlab_custom_script_config_yaml.py                                     ─╯\n\nTask name: 'Backend' Issue URL to update: https://gitlab.com/group1/project2/-/issues/1\n\nTask name: 'Frontend' Issue URL to update: https://gitlab.com/group2/project4/-/issues/2\n\n```\n\n\n\n### CI/CD code linting for different Python versions\n\n\nAll code examples in this blog post have been tested with Python 3.8, 3.9, 3.10 and 3.11, using [parallel matrix builds in GitLab CI/CD](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/.gitlab-ci.yml) and pyflakes for code linting. Automating the tests helps focus on development, and ensuring that the target platforms support the language features. Some Linux distributions do not provide Python 3.11 yet for example, and Python language features cannot be used or may need an alternative implementation.\n\n\n```yaml\n\ninclude:\n  - template: Security/SAST.gitlab-ci.yml\n  - template: Dependency-Scanning.gitlab-ci.yml\n  - template: Secret-Detection.gitlab-ci.yml\n\nstages:\n  - lint\n  - test\n\n.python-req:\n  image: python:$VERSION\n  script:\n    - pip install -r requirements_dev.txt\n  parallel:\n    matrix:\n      - VERSION: ['3.8', '3.9', '3.10', '3.11']   # https://hub.docker.com/_/python\n\nlint-python:\n  extends: .python-req\n  stage: lint\n  script:\n    - !reference [.python-req, script]\n    - pyflakes .\n\nsast:\n  stage: test\n\n```\n\n\n## Optimize code and performance\n\n\n- [Lazy objects](#lazy-objects)\n\n- [Object-oriented programming](#object-oriented-programming)\n\n\n### Lazy objects\n\n\nWhen working with objects that do not immediately need all attributes loaded, you can specify the [`lazy=True`](https://python-gitlab.readthedocs.io/en/stable/api-usage.html#lazy-objects) attribute to not invoke an API call immediately. A follow-up method call will then invoke the required API calls.\n\n\n\n```python\n\n# Lazy object, no API call\n\nproject = gl.projects.get(PROJECT_ID, lazy=True)\n\n\ntry:\n    print(\"Trying to access 'snippets_enabled' on a lazy loaded project object. This will throw an exception that we capture.\")\n    print(\"Project settings: snippets_enabled={b}\".format(b=project.snippets_enabled))\nexcept Exception as e:\n    print(\"Accessing lazy loaded object failed: {e}\".format(e=e))\n\nproject.snippets_enabled = True\n\n\nproject.save() # This creates an API call\n\n\nprint(\"\\nLazy object was loaded after save() call.\")\n\nprint(\"Project settings: snippets_enabled={b}\".format(b=project.snippets_enabled))\n\n\n```\n\n\nExecuting the [`python_gitlab_lazy_objects.py`](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_lazy_objects.py) script shows that the lazy object did not fire an API call, thus throwing an exception when accessing the project setting `snippets_enabled`. To show that the object still can be managed, the code catches the exception to proceed with updating the setting locally, and calling `project.save()` to persist the change and call the API update.\n\n\n```shell\n\n$ python3 python_gitlab_lazy_objects.py                                                ─╯\n\nTrying to access 'snippets_enabled' on a lazy loaded project object. This will throw an exception that we capture.\n\nAccessing lazy loaded object failed: 'Project' object has no attribute 'snippets_enabled'\n\n\nIf you tried to access object attributes returned from the server,\n\nnote that \u003Cclass 'gitlab.v4.objects.projects.Project'> was created as\n\na `lazy` object and was not initialized with any data.\n\n\nLazy object was loaded after save() call.\n\nProject settings: snippets_enabled=True\n\n```\n\n\n### Object-oriented programming\n\n\nFor better code quality, it makes sense to follow object-oriented programming and create classes that store attributes, provide methods, and enable better unit testing. The [storage analyzer tool](https://gitlab.com/gitlab-de/gitlab-storage-analyzer) was developed to create a summary of projects that consume lots storage, for example CI/CD job artifacts. By inspecting the [Git history](https://gitlab.com/gitlab-de/gitlab-storage-analyzer/-/commits/main), you can learn from the different iterations to a first working version.\n\n\nThe following example is a trimmed version which shows how to initialize the class `GitLabUseCase`, add helper functions for logging and JSON pretty-printing, and print all project attributes.\n\n\n```python\n\n#!/usr/bin/env python\n\n\nimport gitlab\n\nimport os\n\nimport sys\n\nimport json\n\n\n# Print an error message with prefix, and exit immediately with an error code.\n\ndef error(text):\n    logger(\"ERROR\", text)\n    sys.exit(1)\n\n# Log a line with a given prefix (e.g. INFO)\n\ndef logger(prefix, text):\n    print(\"{prefix}: {text}\".format(prefix=prefix, text=text))\n\n# Return a pretty-printed JSON string with indent of 4 spaces\n\ndef render_json_output(data):\n    return json.dumps(data, indent=4, sort_keys=True)\n\n# Class definition\n\nclass GitLabUseCase(object):\n    # Initializer to set all required parameters\n    def __init__(self, verbose, gl_server, gl_token, gl_project_id):\n        self.verbose = verbose\n        self.gl_server = gl_server\n        self.gl_token = gl_token\n        self.gl_project_id = gl_project_id\n\n    # Debug logger, controlled via verbose parameter\n    def log_debug(self, text):\n        if self.verbose:\n            print(\"DEBUG: {d}\".format(d=text))\n\n    # Connect to the GitLab server and store the connection handle\n    def connect(self):\n        self.log_debug(\"Connecting to GitLab API at {s}\".format(s=self.gl_server))\n        # Supports personal/project/group access token\n        # https://docs.gitlab.com/ee/api/index.html#personalprojectgroup-access-tokens\n        self.gl = gitlab.Gitlab(self.gl_server, private_token=self.gl_token)\n\n    # Use the stored connection handle to fetch a project object by id,\n    # and print its attribute with JSON pretty-print.\n    def print_project_attributes(self):\n        project = self.gl.projects.get(self.gl_project_id)\n        print(render_json_output(project.attributes))\n\n\n## main\n\nif __name__ == '__main__':\n    # Fetch configuration from environment variables.\n    # The second parameter specifies the default value when not provided.\n    gl_verbose = os.environ.get('GL_VERBOSE', False)\n    gl_server = os.environ.get('GL_SERVER', 'https://gitlab.com')\n\n    gl_token = os.environ.get('GL_TOKEN')\n\n    if not gl_token:\n        error(\"Please specifiy the GL_TOKEN env variable\")\n\n    gl_project_id = os.environ.get('GL_PROJECT_ID', 42491852) # https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python\n\n    # Instantiate new object and run methods\n    gl_use_case = GitLabUseCase(gl_verbose, gl_server, gl_token, gl_project_id)\n    gl_use_case.connect()\n    gl_use_case.print_project_attributes()\n```\n\n\nRunning the [script](https://gitlab.com/gitlab-de/use-cases/gitlab-api/gitlab-api-python/-/blob/main/python_gitlab_oop_helpers.py) with the `GL_PROJECT_ID` environment variable pretty-prints the project attributes as JSON on the terminal.\n\n\n![Example script that pretty-prints the project object attributes as JSON](/images/blogimages/efficient-devsecops-workflows-python-gitlab-handson/python_gitlab_oop_example_terminal_output_project_attributes.png){: .shadow}\n\n\n## More use cases\n\n\nBetter performance with API requests can be achieved by looking into parallelization and threading in Python. Users have been testing the storage analyzer script, and provided feedback to optimize the performance for the single-threaded script by using tasks and [Python threading](https://realpython.com/intro-to-python-threading/), similar to [this community project](https://gitlab.com/thelabnyc/gitlab-storage-cleanup). I might follow up on this topic in a future blog post, there are many more great use cases to cover using python-gitlab.\n\n\nThere is so much more to learn, here are a few examples from the GitLab community forum that could not make it into this blog post:\n\n\n* [Fetch review app environment URL from Merge Request](https://forum.gitlab.com/t/fetch-review-app-environment-url-from-merge-request/71335/2)\n\n* [Project visibility, project features, permissions](https://forum.gitlab.com/t/project-visibility-project-features-permissions-settings-api/32242)\n\n* [Download GitLab CI/CD job artifacts using Python](https://forum.gitlab.com/t/download-gitlab-ci-jobs-artifacts-using-python/25436/$)\n\n\n## Conclusion\n\n\nThe python-gitlab library helps to abstract raw REST API calls, and to keep access to attributes, functions and objects short and relatively easy. There are many use cases that can be solved efficiently. Alternative programming language libraries for the GitLab REST API are available [in the API clients section here](/partners/technology-partners/#api-clients).\n\n\nThe [GitLab Community Forum](https://forum.gitlab.com/) is a great place to collaborate on use cases and questions about possible solutions or code snippets. We'd love to hear from you about your use cases and challenges using the python-gitlab library.\n\n\nShoutout to the python-gitlab maintainers and contributors, developing this fantastic API library for many years now! If this blog post and the python-gitlab library helped you get more efficient, please consider [contributing to python-gitlab](https://python-gitlab.readthedocs.io/en/stable/#contributing). When there is a GitLab API feature missing, look into [contributing to GitLab](https://about.gitlab.com/community/contribute/), too. Thank you!\n\n\n\nCover image by [David Clode](https://unsplash.com/@davidclode) on [Unsplash](https://unsplash.com/photos/cxMJYcuCLEA)\n\n{: .note}\n",[232,9,851,495],{"slug":1730,"featured":6,"template":700},"efficient-devsecops-workflows-hands-on-python-gitlab-api-automation","content:en-us:blog:efficient-devsecops-workflows-hands-on-python-gitlab-api-automation.yml","Efficient Devsecops Workflows Hands On Python Gitlab Api Automation","en-us/blog/efficient-devsecops-workflows-hands-on-python-gitlab-api-automation.yml","en-us/blog/efficient-devsecops-workflows-hands-on-python-gitlab-api-automation",{"_path":1736,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1737,"content":1743,"config":1749,"_id":1751,"_type":13,"title":1752,"_source":15,"_file":1753,"_stem":1754,"_extension":18},"/en-us/blog/efficient-devsecops-workflows-with-rules-for-conditional-pipelines",{"title":1738,"description":1739,"ogTitle":1738,"ogDescription":1739,"noIndex":6,"ogImage":1740,"ogUrl":1741,"ogSiteName":685,"ogType":686,"canonicalUrls":1741,"schema":1742},"DevSecOps workflows with conditional CI/CD pipeline rules","CI/CD pipelines can be simple or complex, what makes them efficient are CI rules that define when and how they run.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749669673/Blog/Hero%20Images/engineering.png","https://about.gitlab.com/blog/efficient-devsecops-workflows-with-rules-for-conditional-pipelines","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to create efficient DevSecOps workflows with rules for conditional CI/CD pipelines\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2023-06-27\",\n      }",{"title":1744,"description":1739,"authors":1745,"heroImage":1740,"date":1746,"body":1747,"category":785,"tags":1748},"How to create efficient DevSecOps workflows with rules for conditional CI/CD pipelines",[1604],"2023-06-27","CI/CD pipelines can be simple or complex – what makes them efficient are\nrules that define when and how they run. By using rules, you create smarter\nCI/CD pipelines, which increase teams' productivity and allow organizations\nto iterate faster. In this tutorial, you will learn about the different\ntypes of CI/CD pipelines and rules and their use cases.\n\n\n## What is a pipeline?\n\nA pipeline is a top-level component of [continuous\nintegration](https://docs.gitlab.com/ee/ci/introduction/index.html#continuous-integration)\nand [continuous\ndelivery](https://docs.gitlab.com/ee/ci/introduction/index.html#continuous-delivery)/[continuous\ndeployment](https://docs.gitlab.com/ee/ci/introduction/index.html#continuous-deployment),\nand it comprises [jobs](https://docs.gitlab.com/ee/ci/jobs/index.html),\nwhich are lists of tasks to be executed. Jobs are organized in\n[stages](https://docs.gitlab.com/ee/ci/yaml/index.html#stages), which define\nwhen the jobs run.\n\n\nA pipeline can be a [basic\none](https://docs.gitlab.com/ee/ci/pipelines/pipeline_architectures.html#basic-pipelines)\nin which jobs run concurrently in each stage. Pipelines can also be complex,\nlike [parent-child\npipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html#parent-child-pipelines),\n[merge trains](https://docs.gitlab.com/ee/ci/pipelines/merge_trains.html),\n[multi-project\npipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html#multi-project-pipelines),\nor the more advanced [Directed Acyclic Graph\npipelines](https://docs.gitlab.com/ee/ci/directed_acyclic_graph/index.html)\n(DAG).\n\n\n![Complex pipeline showing\ndependencies](https://about.gitlab.com/images/blogimages/2023-06-15-efficient-devsecops-workflows-with-rules-for-conditional-pipelines/complex-pipelines.png)\n\n\nA [gitlab-runner\npipeline](https://gitlab.com/gitlab-org/gitlab-runner/-/pipelines/798871212/)\nshowing job dependencies.\n\n{: .note.text-center}\n\n\n![Directed Acyclic\nGraph](https://about.gitlab.com/images/blogimages/2023-06-15-efficient-devsecops-workflows-with-rules-for-conditional-pipelines/dag-pipelines.png)\n\n\nDirected Acyclic Graph pipeline\n\n{: .note.text-center}\n\n\nUse cases determine how complicated a pipeline can get. A use case might\nrequire testing an application and packaging it into a container; the\npipeline can even further deploy the container to an orchestrator like\nKubernetes or a container registry. Another use case might involve building\napplications that target different platforms with varying dependencies,\nwhich is where DAG pipelines shine.\n\n\n## What are CI/CD rules?\n\nCI/CD rules are the key to managing the flow of jobs in a pipeline. One of\nthe powerful features of GitLab CI/CD is the ability to control when a CI/CD\njob runs, which can depend on context, changes made,\n[workflow](https://docs.gitlab.com/ee/ci/yaml/workflow.html) rules, values\nof CI/CD variables, or custom conditions. Aside from using `rules`, you can\nalso control the flow of CI/CD pipelines using:\n\n\n* [`needs`](https://docs.gitlab.com/ee/ci/yaml/index.html#needs):\nestablishes relationships between jobs and used in DAG pipelines\n\n* [`only`](https://docs.gitlab.com/ee/ci/yaml/index.html#only--except):\ndefines when a job should run\n\n* [`except`](https://docs.gitlab.com/ee/ci/yaml/index.html#only--except):\ndefines when a job should not run\n\n* [`workflow`](https://docs.gitlab.com/ee/ci/yaml/workflow.html): controls\nwhen pipelines are created\n\n\n`only` and `except` should not be used with `rules` as this can lead to\nunexpected behavior. It is recommended to use `rules`, learn more in the\nfollowing sections.\n\n\n## What is the `rules` feature?\n\n`rules` determine when and if a job runs in a pipeline. If you have multiple\nrules defined, they are all evaluated in order until a matching rule is\nfound and the job is executed according to the specified configuration.\n\n\n[Rules](https://docs.gitlab.com/ee/ci/yaml/#rules) can be defined using the\nkeywords: `if`, `changes`, `exists`, `allow_failure`, `variables`, `when`\nand `needs`.\n\n\n### `rules:if`\n\nThe `if` keyword evaluates if a job should be added to a pipeline. The\nevaluation is done based on the values of [CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/index.html) defined in\nthe scope of the job or pipeline and [predefined CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html).\n\n\n```yaml\n\njob:\n  script:\n    - echo $(date)\n  rules:\n    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $CI_DEFAULT_BRANCH\n```\n\n\nIn the CI/CD script above, the job prints the current date and time with the\n`echo` command. The job is only executed if the source branch of a merge\nrequest (`CI_MERGE_REQUEST_SOURCE_BRANCH_NAME`) is the same as the project's\ndefault branch (`CI_DEFAULT_BRANCH`) in a [merge request\npipeline](https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html).\nYou can use the `==` and `!=` operators for comparison, while `=~` and `!~`\nallow you to compare a variable to a regular expression. You can combine\nmultiple expressions using the `&&` (AND), `||` (OR) operators, and\nparentheses for grouping expressions.\n\n\n### `rules:changes`\n\nWith the `changes` keyword, you can watch for changes to certain files or\nfolders for a job to execute. GitLab uses the output of [Git\ndiffstat](https://git-scm.com/docs/git-diff#Documentation/git-diff.txt\n\n\n```yaml\n\njob:\n  script:\n    - terraform plan\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n      changes:\n        - terraform/**/*.tf\n```\n\n\nIn this example, the `terraform plan` is only executed when files with the\n`.tf` extension are changed in the `terraform` folder and its\nsubdirectories. An additional rule ensures the job is executed for [merge\nrequest\npipelines](https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html).\n\n\nThe `changes` rule can look for changes in specific files with `paths`:\n\n\n```yaml\n\njob:\n  script:\n    - terraform plan\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n      changes:\n        paths:\n          - terraform/main.tf\n```\n\n\nChanges to files in a source reference (branch, tag, commit) can also be\ncompared against other references in the Git repository. The CI/CD job will\nonly execute when the source reference differs from the [specified reference\nvalue defined in\n`rules:changes:compare_to`](https://docs.gitlab.com/ee/ci/yaml/#ruleschangescompare_to).\nThis value can be a Git commit SHA, tag, or branch name. The following\nexample compares the source reference to the current `production` branch\n(`refs/head/production`).\n\n\n```yaml\n\njob:\n  script:\n    - terraform plan\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n      changes:\n        paths:\n          - terraform/main.tf\n        compare_to: 'refs/head/production'\n```\n\n\n### `rules:exists`\n\nLike `changes`, you can execute CI/CD jobs only when specific files exist\n[using `rules:exists`\nrules](https://docs.gitlab.com/ee/ci/yaml/#rulesexists). For example, you\ncan run a job that checks whether a `Gemfile.lock` file exists. The\nfollowing example audits a Ruby project for vulnerable versions of gems or\ninsecure gem sources using the [bundler-audit\nproject](https://github.com/rubysec/bundler-audit).\n\n\n```yaml\n\njob:\n  script:\n    - bundle-audit check --format json --output bundle-audit.json\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n      changes:\n        exits:\n          - Gemfile.lock\n```\n\n\n### `rules:allow_failure`\n\nThere are scenarios where the failure of a job should not affect the\nfollowing jobs and stages of the pipeline. This can be useful in use cases\nwhere non-blocking tasks are required as part of a project but don't impact\nthe project in any way. The [`rules:allow_failure`\nrule](https://docs.gitlab.com/ee/ci/yaml/#rulesallow_failure) can be set to\n`true` or `false`. It defaults to `false` implicitly when the rule is not\nspecified.\n\n\n```yaml\n\njob:\n  script:\n    - bundle-audit check --format json --output bundle-audit.json\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\" && $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED == \"false\"\n      changes:\n        exits:\n          - Gemfile.lock\n      allow_failure: true\n```\n\n\nIn this example, the job can fail only if a merge request event triggers the\npipeline and the target branch is not protected.\n\n\n### `rules:needs`\n\nDisabled by fault,\n[`rules:needs`](https://docs.gitlab.com/ee/ci/yaml/#rulesneeds) was\nintroduced in [GitLab\n16](https://about.gitlab.com/releases/2023/05/22/gitlab-16-0-released/) and\ncan be enabled with the `introduce_rules_with_needs` [feature\nflag](https://docs.gitlab.com/ee/user/feature_flags.html).\n[`needs`](https://docs.gitlab.com/ee/ci/yaml/index.html#needs) is used to\nexecute jobs out of order without waiting for other jobs in a stage to\ncomplete. When used with `rules`, it replaces the job's `needs`\nspecification when the set conditions are met.\n\n\n```yaml\n\nstages:\n  - build\n  - qa\n  - deploy\n\nbuild-dev:\n  stage: build\n  rules:\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n  script: echo \"Building dev version...\"\n\nbuild-prod:\n  stage: build\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n  script: echo \"Building production version...\"\n\nqa-checks:\n  stage: qa\n  script:\n    - echo \"Running QA checks before publishing to Production....\"\n\ndeploy:\n  stage: deploy\n  needs: ['build-dev']\n  rules:\n    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n      needs: ['build-prod', 'qa-checks']\n    - when: on_success # Run the job in other cases\n  script: echo \"Deploying application.\"\n\n```\n\n\nIn the example above, the deploy job has the `build-dev` job as a dependency\nbefore it runs; however, when the commit branch is the project's default\nbranch, its dependency changes to `build-prod` and `qa-checks`. This can\nallow for extra checks to be implemented based on context.\n\n\n### `rules:variables`\n\nIn some situations, you only need certain variables in specific conditions,\nor their values change based on content; you can use the\n[`rules:variables`](https://docs.gitlab.com/ee/ci/yaml/#rulesvariables) rule\nto define variables when specific conditions are met. This also allows to\ncreate more dynamic CI/CD execution workflows.\n\n\n```\n\njob:\n  variables:\n    DEPLOY_VERSION: \"dev\"\n  rules:\n    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n      variables:\n        DEPLOY_VERSION: \"stable\"\n  script:\n    - echo \"Deploying $DEPLOY_VERSION version\"\n```\n\n\n### `workflow:rules`\n\nSo far, we have looked at controlling when jobs run in a pipeline using the\n`rules` keyword. Sometimes, you want to control how the entire pipeline\nbehaves: That's where [`workflow:rules` provide a powerful\noption](https://docs.gitlab.com/ee/ci/yaml/#workflowrules). `workflow:rules`\nare evaluated before jobs and take precedence over the job rules. For\nexample, if a job has rules that allow it to run against a specific branch,\nbut the workflow rules set jobs running against the branch to `when: never`,\nthe jobs will not run.\n\n\nAll the features of `rules` mentioned in the previous sections work for\n`workflow:rules`.\n\n\n```yaml\n\nworkflow:\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"schedule\"\n      when: never\n    - if: $CI_PIPELINE_SOURCE == \"push\"\n      when: never\n    - when: always\n```\n\n\nIn the example above, the CI/CD pipeline runs except when a schedule or push\nevent is triggered.\n\n\n## Use cases for CI/CD rules\n\nIn the previous section, we looked at different ways of using the `rules`\nfeature of GitLab CI/CD. In this section, we will explore practical use\ncases.\n\n\n### Developer experience\n\nOne of the benefits of a DevSecOps platform is to allow developers to focus\non what they do best: writing their code and doing as little operations as\npossible. A company's DevOps or Platform team can create CI/CD templates for\ndifferent stages of their development lifecycle and use rules to add CI/CD\njobs to handle specific tasks based on their technology stack. A developer\nonly needs to include a default CI/CD script and pipelines are automatically\ncreated based on files detected, refs used, or defined variables, leading to\nincreased productivity.\n\n\n### Security and quality assurance\n\nA major function of CI/CD pipelines is to catch bugs or vulnerabilities\nbefore they are deployed into production infrastructure. Using CI/CD rules,\nsecurity and quality assurance teams can dynamically run extra checks on\nchanges introduced when certain factors are introduced. For example, malware\nscans can be added when new file extensions not in an approved list are\ndetected, or more advanced performance tests are automatically added when a\ncertain level of change has been introduced to the codebase. With GitLab's\nbuilt-in security, including security in your pipelines can be done with\njust a few lines of code.\n\n\n```yaml\n\ninclude:\n  # Static\n  - template: Jobs/Container-Scanning.gitlab-ci.yml\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml\n  - template: Jobs/SAST.gitlab-ci.yml\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n  - template: Jobs/SAST-IaC.gitlab-ci.yml\n  - template: Jobs/Code-Quality.gitlab-ci.yml\n  - template: Security/Coverage-Fuzzing.gitlab-ci.yml\n  # Dynamic\n  - template: Security/DAST.latest.gitlab-ci.yml\n  - template: Security/BAS.latest.gitlab-ci.yml\n  - template: Security/DAST-API.latest.gitlab-ci.yml\n  - template: API-Fuzzing.latest.gitlab-ci.yml\n```\n\n\n### Automation\n\nThe power of CI/CD rules shines through in the (nearly) limitless\npossibilities of automating your CI/CD pipelines. GitLab\n[AutoDevOps](https://docs.gitlab.com/ee/topics/autodevops/) is an example.\nIt uses an opinionated best-practice collection of [GitLab CI/CD\ntemplates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates)\nand rules to detect the technology stack used. AutoDevOps creates relevant\njobs that take your application all the way to production from a push. You\ncan review the [AutoDevOps\ntemplate](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml)\nto learn how it leverages CI/CD rules for greater efficiency.\n\n\n### Using CI/CD components\n\nGrowth comes with several iterations of work and creating best practices.\nWhile building CI/CD pipelines, your DevOps team would have made several\nCI/CD scripts that they repurpose across pipelines using the\n[`include`](https://docs.gitlab.com/ee/ci/yaml/#include) keyword. In [GitLab\n16](https://about.gitlab.com/releases/2023/05/22/gitlab-16-0-released/),\nGitLab [introduced CI/CD\nComponents](https://about.gitlab.com/releases/2023/05/22/gitlab-16-0-released/#cicd-components),\nan experimental feature that allows your team to create reusable CI/CD\ncomponents and publish them as a catalog that can be used to build smarter\nCI/CD pipelines rapidly. You can learn more [about using CI/CD\ncomponents](https://docs.gitlab.com/ee/ci/components/) and the [component\ncatalog\ndirection](https://about.gitlab.com/direction/verify/component_catalog/).\n\n\nGitLab CI/CD enables you to run smarter pipelines, and it does so together\nwith [GitLab Duo, AI-powered workflows](/gitlab-duo/) to help you build more\nsecure software, faster.\n",[9,787,788,851,495],{"slug":1750,"featured":6,"template":700},"efficient-devsecops-workflows-with-rules-for-conditional-pipelines","content:en-us:blog:efficient-devsecops-workflows-with-rules-for-conditional-pipelines.yml","Efficient Devsecops Workflows With Rules For Conditional Pipelines","en-us/blog/efficient-devsecops-workflows-with-rules-for-conditional-pipelines.yml","en-us/blog/efficient-devsecops-workflows-with-rules-for-conditional-pipelines",{"_path":1756,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1757,"content":1763,"config":1768,"_id":1770,"_type":13,"title":1771,"_source":15,"_file":1772,"_stem":1773,"_extension":18},"/en-us/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces",{"title":1758,"description":1759,"ogTitle":1758,"ogDescription":1759,"noIndex":6,"ogImage":1760,"ogUrl":1761,"ogSiteName":685,"ogType":686,"canonicalUrls":1761,"schema":1762},"Enable secure sudo access for GitLab Remote Development workspaces","Learn how to allow support for sudo commands using Sysbox, Kata Containers, and user namespaces in this easy-to-follow tutorial.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675033/Blog/Hero%20Images/blog-image-template-1800x945.png","https://about.gitlab.com/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Enable secure sudo access for GitLab Remote Development workspaces\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Vishal Tak\"}],\n        \"datePublished\": \"2024-11-20\",\n      }",{"title":1758,"description":1759,"authors":1764,"heroImage":1760,"date":1484,"body":1765,"category":830,"tags":1766},[1179],"A development environment often requires sudo permissions to install, configure, and use dependencies during runtime. GitLab now allows secure sudo access for [GitLab Remote Development workspaces](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/). This tutorial shows you how to enable GitLab workspace users to securely use sudo commands to perform common tasks.\n\n## The challenge\n\nFor the sake of this article, say your project is as simple as the below code.\n\n```\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"log/slog\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\t// Set up JSON logger\n\tlogFile, err := os.OpenFile(\"server.log\", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer logFile.Close()\n\n\tjsonHandler := slog.NewJSONHandler(logFile, nil)\n\tlogger := slog.New(jsonHandler)\n\tslog.SetDefault(logger)\n\n\t// Define handlers\n\thttp.HandleFunc(\"/path1\", handleRequest)\n\thttp.HandleFunc(\"/path2\", handleRequest)\n\n\t// Start server\n\tslog.Info(\"Starting server on :3000\")\n\terr = http.ListenAndServe(\":3000\", nil)\n\tif err != nil {\n\t\tslog.Error(\"Server failed to start\", \"error\", err)\n\t}\n}\n\nfunc handleRequest(w http.ResponseWriter, r *http.Request) {\n\tdata := make(map[string]interface{})\n\tfor k, v := range r.Header {\n\t\tdata[k] = v\n\t}\n\n\tdata[\"method\"] = r.Method\n\tdata[\"url\"] = r.URL.String()\n\tdata[\"remote_addr\"] = r.RemoteAddr\n\n\tresponse, err := json.MarshalIndent(data, \"\", \"  \")\n\tif err != nil {\n\t\tslog.Error(\"Failed to marshal metadata\", \"error\", err)\n\t\thttp.Error(w, \"Internal Server Error\", http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\t// Log the metadata\n\tslog.Info(\"Request received\",\n\t\t\"path\", r.URL.Path,\n\t\t\"response\", string(response),\n\t)\n\n\t// Write response\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tw.Write(response)\n}\n```\n\nThis code starts an HTTP server on port 3000, exposes two paths: `path1` and `path2`. Each HTTP request received is logged to a file `server.log`.\n\nLet's run this code with `go run main.go` and generate some requests.\n\n```\ni=1\nwhile [ \"$i\" -le 100 ]; do\n  echo \"Iteration $i\"\n\n  if [ $((random_number % 2)) -eq 0 ]; then\n    curl \"localhost:3000/path1\"\n  else\n    curl \"localhost:3000/path2\"\n  fi\n\n  i=$((i + 1))\ndone\n```\n\nAs you work on this application, you realize the need to analyze the logs to debug an issue. You look at the log file and it is long to parse with a simple glance. You remember there is a handy tool, [jq](https://jqlang.github.io/jq/), which parses JSON data. But your workspace does not have it installed.\n\nYou want to install `jq` through the package manager for this workspace only.\n\n```\nsudo apt update\nsudo apt install jq\n```\n\nThe output is:\n\n```\nsudo: The \"no new privileges\" flag is set, which prevents sudo from running as root.\nsudo: If sudo is running in a container, you may need to adjust the container configuration to disable the flag.\n```\n\nThis happens because GitLab workspaces explicitly disallows `sudo` access to prevent privilege escalation on the Kubernetes host.\n\nNow, there is a more secure way to run `sudo` commands in a workspace.\n\n## How sudo access works\n\nThat is exactly what we have [unlocked](https://docs.gitlab.com/ee/user/workspace/configuration.html#configure-sudo-access-for-a-workspace) in the 17.4 release of GitLab.\n\nYou can configure secure sudo access for workspaces using any of the following options:\n\n- Sysbox  \n- Kata Containers  \n- User namespaces\n\nWe will set up three GitLab agents for workspaces to demonstrate each option.\n\n### Sysbox\n\n[Sysbox](https://github.com/nestybox/sysbox) is a container runtime that improves container isolation and enables containers to run the same workloads as virtual machines.\n\nTo configure sudo access for a workspace with Sysbox:\n\n1. In the Kubernetes cluster, [install Sysbox](https://github.com/nestybox/sysbox#installation).\n2. In the GitLab agent for workspaces, set the following config:\n\n```\nremote_development:\n  enabled: true\n  dns_zone: \"sysbox-update.me.com\"\n  default_runtime_class: \"sysbox-runc\"\n  allow_privilege_escalation: true\n  annotations:\n    \"io.kubernetes.cri-o.userns-mode\": \"auto:size=65536\"\n```\n\n3. Add other settings in the agent config as per your requirements. [GitLab agent for workspaces settings](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#workspace-settings) for more information about individual settings.  \n4. Allow the agent to be used for workspaces in a group. See the [documentation](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#allow-a-cluster-agent-for-workspaces-in-a-group) for more information.  \n5. Update GitLab Workspaces Proxy to serve traffic for the domain used in the above agent configuration. See [Tutorial: Set up the GitLab workspaces proxy](https://docs.gitlab.com/ee/user/workspace/set_up_workspaces_proxy.html) for more information.\n\n### Kata Containers\n\n[Kata Containers](https://github.com/kata-containers/kata-containers) is a standard implementation of lightweight virtual machines that perform like containers but provide the workload isolation and security of virtual machines.\n\nTo configure sudo access for a workspace with Kata Containers:\n\n1. In the Kubernetes cluster, [install Kata Containers](https://github.com/kata-containers/kata-containers/tree/main/docs/install).  \n2. In the GitLab agent for workspaces, set the following config:\n\n```\nremote_development:\n  enabled: true\n  dns_zone: \"kata-update.me.com\"\n  default_runtime_class: \"kata-qemu\"\n  allow_privilege_escalation: true\n```\n\n3. Add other settings in the agent config as per your requirements. [GitLab agent for workspaces settings](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#workspace-settings) for more information about individual settings.  \n4. Allow the agent to be used for workspaces in a group. See the [documentation](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#allow-a-cluster-agent-for-workspaces-in-a-group) for more information.  \n5. Update GitLab Workspaces Proxy to serve traffic for the domain used in the above agent configuration. See [Tutorial: Set up the GitLab workspaces proxy](https://docs.gitlab.com/ee/user/workspace/set_up_workspaces_proxy.html) for more information.\n\n### User namespaces\n\n[User namespaces](https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/) isolate the user running inside the container from the user on the host.\n\nTo configure sudo access for a workspace with user namespaces:\n\n1. In the Kubernetes cluster, [configure user namespaces](https://kubernetes.io/blog/userns-beta/).  \n2. In the GitLab agent for workspaces, set the following config:\n\n```\nremote_development:\n  enabled: true\n  dns_zone: \"userns-update.me.com\"\n  use_kubernetes_user_namespaces: true\n  allow_privilege_escalation: true\n```\n\n3. Add other settings in the agent config as per your requirements. [GitLab agent for workspaces settings](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#workspace-settings) for more information about individual settings.  \n4. Allow the agent to be used for workspaces in a group. See the [documentation](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#allow-a-cluster-agent-for-workspaces-in-a-group) for more information.  \n5. Update GitLab Workspaces Proxy to serve traffic for the domain used in the above agent configuration. See [Tutorial: Set up the GitLab workspaces proxy](https://docs.gitlab.com/ee/user/workspace/set_up_workspaces_proxy.html) for more information.\n\nSetting up a Kubernetes cluster with user namespaces configured is challenging since it is behind a beta feature gate in Kubernetes Version 1.31.0. This means it is not yet possible to configure such a cluster on the major cloud providers because they don't provide a mechanism to enable feature gates in their managed Kubernetes offering. Here is an example of [configuring a simple Kuberenetes cluster using `kubeadm`](https://gitlab.com/gitlab-org/gitlab/-/issues/468290#note_1959300036).\n\n### Create a workspace\n\nIf you now create a workspace with these agents and try installing `jq` through a package manager, it should succeed!\n\nYou can analyze the logs using `jq`. Say you wanted to inspect the log entries where the path is `/path1`, you can run:\n\n```\njq 'select(.path == \"/path1\")' server.log\n```\n\nThe output is:\n\n```\n{\n  \"time\": \"2024-10-31T12:04:38.474806+05:30\",\n  \"level\": \"INFO\",\n  \"msg\": \"Request received\",\n  \"path\": \"/path1\",\n  \"response\": \"{\\n  \\\"Accept\\\": [\\n    \\\"*/*\\\"\\n  ],\\n  \\\"User-Agent\\\": [\\n    \\\"curl/8.7.1\\\"\\n  ],\\n  \\\"method\\\": \\\"GET\\\",\\n  \\\"remote_addr\\\": \\\"[::1]:61246\\\",\\n  \\\"url\\\": \\\"/path1\\\"\\n}\"\n}\n{\n  \"time\": \"2024-10-31T12:06:22.397453+05:30\",\n  \"level\": \"INFO\",\n  \"msg\": \"Request received\",\n  \"path\": \"/path1\",\n  \"response\": \"{\\n  \\\"Accept\\\": [\\n    \\\"*/*\\\"\\n  ],\\n  \\\"User-Agent\\\": [\\n    \\\"curl/8.7.1\\\"\\n  ],\\n  \\\"method\\\": \\\"GET\\\",\\n  \\\"remote_addr\\\": \\\"[::1]:61311\\\",\\n  \\\"url\\\": \\\"/path1\\\"\\n}\"\n}\n{\n  \"time\": \"2024-10-31T12:19:34.974354+05:30\",\n  \"level\": \"INFO\",\n  \"msg\": \"Request received\",\n  \"path\": \"/path1\",\n  \"response\": \"{\\n  \\\"Accept\\\": [\\n    \\\"*/*\\\"\\n  ],\\n  \\\"User-Agent\\\": [\\n    \\\"curl/8.7.1\\\"\\n  ],\\n  \\\"method\\\": \\\"GET\\\",\\n  \\\"remote_addr\\\": \\\"[::1]:61801\\\",\\n  \\\"url\\\": \\\"/path1\\\"\\n}\"\n}\n```\n\n## Get started today\n\nLearn even more with our [Configure sudo access for a workspace documentation](https://docs.gitlab.com/ee/user/workspace/configuration.html#configure-sudo-access-for-a-workspace). See [GitLab agent for workspaces settings](https://docs.gitlab.com/ee/user/workspace/gitlab_agent_configuration.html#workspace-settings) for details on individual settings.\n\n> New to GitLab Remote Development? Here is a [quickstart guide](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/) to get you up to speed.",[830,9,1767,495,765],"remote work",{"slug":1769,"featured":90,"template":700},"enable-secure-sudo-access-for-gitlab-remote-development-workspaces","content:en-us:blog:enable-secure-sudo-access-for-gitlab-remote-development-workspaces.yml","Enable Secure Sudo Access For Gitlab Remote Development Workspaces","en-us/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces.yml","en-us/blog/enable-secure-sudo-access-for-gitlab-remote-development-workspaces",{"_path":1775,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1776,"content":1779,"config":1785,"_id":1787,"_type":13,"title":1788,"_source":15,"_file":1789,"_stem":1790,"_extension":18},"/en-us/blog/enhance-application-quality-with-ai-powered-test-generation",{"noIndex":6,"title":1777,"description":1778},"Enhance application quality with AI-powered test generation","Learn how GitLab Duo with Amazon Q improves the QA process by automatically generating comprehensive unit tests.",{"title":1777,"description":1778,"authors":1780,"heroImage":1781,"date":1782,"body":1783,"category":718,"tags":1784},[715],"https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659604/Blog/Hero%20Images/Screenshot_2024-11-27_at_4.55.28_PM.png","2025-07-03","You know how critical application quality is to your customers and reputation. However, ensuring that quality through comprehensive testing can feel like an uphill battle. You're dealing with time-consuming manual processes, inconsistent test coverage across your team, and those pesky issues that somehow slip through the cracks. It's frustrating when your rating drops because quality assurance becomes a bottleneck rather than a safeguard.\n\nHere's where [GitLab Duo with Amazon Q ](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/), which delivers agentic AI throughout the software development lifecycle for AWS customers, can help transform your QA process. This AI-powered capability can automatically generate comprehensive unit tests for your code, dramatically accelerating your quality assurance workflow. Instead of spending hours writing tests manually, you can let AI analyze your code and create tests that ensure optimal coverage and consistent quality across your entire application.\n\n## How GitLab Duo with Amazon Q works\n\nSo how does this work? Let's walk through the process together.\nWhen you're working on a new feature, you start by selecting the Java class you've added to your project through a merge request. You simply navigate to your merge request and click on the \"Changes\" tab to see the new code you've added.\n\nNext, you invoke Amazon Q by entering a quick action command. All you need to do is type `/q test` in the issue comment box. It's that simple – just a forward slash, the letter \"q\", and the word \"test\".\n\nOnce you hit enter, Amazon Q springs into action. It analyzes your selected code, understanding its structure, logic, and purpose. The AI examines your class methods, dependencies, and potential edge cases to determine what tests are needed.\n\nWithin moments, Amazon Q generates comprehensive unit test coverage for your new class. It creates tests that cover not just the happy path, but also edge cases and error conditions you might have overlooked. The generated tests follow your project's existing patterns and conventions, ensuring they integrate seamlessly with your codebase.\n\n## Why use GitLab Duo with Amazon Q?\n\nHere's the bottom line: You started with a critical challenge – maintaining high-quality applications while dealing with time constraints and inconsistent testing practices. GitLab Duo with Amazon Q addresses this by automating the test generation process, ensuring optimal code coverage and consistent testing standards. The result? Issues are detected before deployment, your applications maintain their quality, and you can develop software faster without sacrificing reliability.\n\nKey benefits of this feature:\n\n* Significantly reduces time spent writing unit tests\n* Ensures comprehensive test coverage across your codebase\n* Maintains consistent testing quality across all team members\n* Catches issues before they reach production\n* Accelerates your overall development velocity\n\nReady to see this game-changing feature in action? Watch how GitLab Duo with Amazon Q can transform your quality assurance process:\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/pxlYJVcHY28?si=MhIz6lnHxc6kFhlL\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## Get started with GitLab Duo with Amazon Q today\n\nWant to learn more about GitLab Duo with Amazon Q? Visit the [GitLab and AWS partner page](https://about.gitlab.com/partners/technology-partners/aws/) for detailed information.\n\n## Agentic AI resources\n- [Agentic AI guides and resources](https://about.gitlab.com/blog/agentic-ai-guides-and-resources/)\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n- [GitLab Duo with Amazon Q: Agentic AI optimized for AWS generally available](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/)\n- [GitLab Duo with Amazon Q documentation](https://docs.gitlab.com/user/duo_amazon_q/)",[720,694,1057,851,9,810],{"featured":90,"template":700,"slug":1786},"enhance-application-quality-with-ai-powered-test-generation","content:en-us:blog:enhance-application-quality-with-ai-powered-test-generation.yml","Enhance Application Quality With Ai Powered Test Generation","en-us/blog/enhance-application-quality-with-ai-powered-test-generation.yml","en-us/blog/enhance-application-quality-with-ai-powered-test-generation",{"_path":1792,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1793,"content":1799,"config":1805,"_id":1807,"_type":13,"title":1808,"_source":15,"_file":1809,"_stem":1810,"_extension":18},"/en-us/blog/enhance-application-security-with-gitlab-hackerone",{"title":1794,"description":1795,"ogTitle":1794,"ogDescription":1795,"noIndex":6,"ogImage":1796,"ogUrl":1797,"ogSiteName":685,"ogType":686,"canonicalUrls":1797,"schema":1798},"Enhance application security with GitLab + HackerOne","Learn about the GitLab + HackerOne partnership and how to easily implement an integration that improves your organization’s application security posture.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097503/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2810%29_5ET24Q6i8ihqrAOkge7a1R_1750097503214.png","https://about.gitlab.com/blog/enhance-application-security-with-gitlab-hackerone","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Enhance application security with GitLab + HackerOne\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-04-03\",\n      }",{"title":1794,"description":1795,"authors":1800,"heroImage":1796,"date":1801,"body":1802,"category":830,"tags":1803},[1444],"2025-04-03","Security can no longer be an afterthought in the development process.\nOrganizations need robust solutions that integrate security throughout the\nentire software development lifecycle. This is where the partnership between\nHackerOne and GitLab creates a compelling combination for modern application\ndevelopment teams.\n\n\nGitLab, the comprehensive, AI-powered DevSecOps platform, and HackerOne, the\nleading crowd-sourced security platform, have established a partnership that\nbrings together the best of both worlds: GitLab's streamlined DevSecOps\nworkflow and HackerOne's powerful vulnerability management capabilities.\n\n\nIn this tutorial, you'll learn how to enhance developer productivity and\nyour security posture by implementing HackerOne's GitLab integration.\n\n\n## An integration that empowers developers\n\n\nHackerOne's GitLab integration is remarkably straightforward, yet powerful.\nWhen security researchers discover vulnerabilities through HackerOne's\nplatform, these findings are automatically converted into GitLab issues.\nThis creates a seamless workflow where:\n\n\n* Security researchers identify vulnerabilities via HackerOne's platform  \n\n* Validated vulnerabilities are automatically converted into GitLab issues  \n\n* Development teams can address these issues directly within their existing\nworkflow  \n\n* Resolution status is synchronized between both platforms\n\n\nYou can start leveraging the benefits of GitLab and HackerOne by using the\n[integration](https://docs.hackerone.com/en/articles/8571227-gitlab-integration)\nto track GitLab issues as references on HackerOne. This integration provides\nbi-directional and seamless data syncing between your HackerOne report and\nGitLab issues, improving alignment between development and security teams\nwhile streamlining security vulnerability processing.\n\n\nTo configure the GitLab integration to sync information between your\nHackerOne report and your Gitlab issue, follow the instructions provided in\n[HackerOne's GitLab integration\ndocumentation](https://docs.hackerone.com/en/articles/10394699-gitlab-setup),\nwhich includes:\n\n\n1. [Setting up an OAuth 2.0\napplication](https://docs.gitlab.com/ee/integration/oauth_provider.html) for\nyour GitLab instance with the provided HackerOne settings  \n\n2. Connecting HackerOne to the newly created OAuth 2.0 on GitLab  \n\n3. Authorizing HackerOne to access the GitLab API  \n\n4. Configuring which GitLab project you would like to escalate HackerOne\nreports to  \n\n5. Selecting the HackerOne fields to map to corresponding GitLab fields  \n\n6. GitLab-to-HackerOne and HackerOne-to-GitLab event configuration\n\n\nOnce the integration is in place, you’ll be able to seamlessly sync data\nbi-directionally between both GitLab and HackerOne. This helps simplify\ncontext-switching and allows vulnerabilities to be tracked with ease\nthroughout both systems. The integration allows for the following features:\n\n\n* **Creating a GitLab Issue from HackerOne:** You can create new GitLab\nissues for reports you receive on HackerOne.  \n\n* **Linking HackerOne reports to existing GitLab tasks.**   \n\n* **Syncing updates from HackerOne to GitLab:** The following updates on a\nreport are synced as a comment to GitLab.  \n  * Report comments  \n  * State changes  \n  * Rewards  \n  * Assignee changes  \n  * Public disclosure  \n  * Close GitLab Issue  \n* **Syncing Updates from GitLab to HackerOne:** The following updates on\nGitLab will be reflected in HackerOne as an internal comment on the\nassociated report:  \n  * Comments  \n  * State changes  \n* **HackerOne severity to GitLab label mapping**: Allows you to set a custom\npriority when escalating a report to GitLab.  \n\n* **Due date mapping:** Allows you to automatically set a custom due date\nbased on the severity of a report.\n\n\n![GitLab + HackerOne adding comments or change the state of the report in\nGitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097510/Blog/Content%20Images/Blog/Content%20Images/sync_aHR0cHM6_1750097509644.png)\n\n\nThese features improve alignment between development and security teams and\nstreamlining security vulnerability processing. To learn more on how the\nintegration works, see the [integration\ndocumentation](https://docs.hackerone.com/en/articles/8571227-gitlab-integration).\n\n\n## A look into HackerOne bug bounty programs\n\n\nHackerOne provides bug bounty programs or cybersecurity initiatives where\nrewards are offered for discovering and reporting vulnerabilities in\ncustomers’ software systems, websites, or applications. Bug bounty programs\nhelp enhance the security of an application by:\n\n\n* Identifying security flaws before malicious actors can exploit them  \n\n* Leveraging diverse expertise from a global community of security\nresearchers  \n\n* Providing a cost-effective way to improve cybersecurity  \n\n* Complementing internal security efforts and traditional penetration\ntesting\n\n\nGitLab utilizes HackerOne’s bug bounty program, allowing security\nresearchers to report vulnerabilities in GitLab applications or\ninfrastructure. This crowdsourced approach helps GitLab identify and address\npotential security issues more effectively.\n\n\n![HackerOne GitLab Bug Bounty\npage](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097510/Blog/Content%20Images/Blog/Content%20Images/hackerone_gitlab_bug_bounty_page_aHR0cHM6_1750097509645.png)\n\n\nBy leveraging HackerOne's platform and the global hacker community,\norganizations can significantly enhance their security posture, identify\nvulnerabilities faster, and stay ahead of potential threats.\n\n\n## Secure applications and improve efficiency with GitLab \n\n\nGitLab provides a complete DevSecOps platform, which enables functionality\nfor the complete software development lifecycle, including security and\ncompliance tools. GitLab supports the following security scanner types:\n\n- Static Application Security Testing (SAST)\n\n- Dynamic Application Security Testing (DAST)\n\n- Container Scanning\n\n- Dependency Scanning\n\n- Infrastructure as Code Scanning\n\n- Coverage-guided Fuzzing\n\n- Web API Fuzzing\n\n\nWith GitLab, you can add security scanning by simply applying a template to\nyour CI/CD pipeline definition file. For example, enabling SAST just takes a\nfew lines of code in the `.gitlab-ci.yml`:\n\n\n```yaml\n\nstage:\n  - test\n\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n```\n\n\nThis will run SAST on the test stage, and [auto-detect the languages\nused](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks)\nin your application. Then, whenever you create a merge request, SAST will\ndetect the vulnerabilities in the diff between the feature branch and the\ntarget branch and provide relevant data on each vulnerability to assist with\nremediation.\n\n\n![NoSQL injection vulnerability seen in\nMR](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097510/Blog/Content%20Images/Blog/Content%20Images/no_sql_injection_vulnerability_mr_view_aHR0cHM6_1750097509647.png)\n\n\nThe results of the SAST scanner can block code from being merged if security\npolicies are applied. Native GitLab users can be set as approvers, allowing\nrequired reviews before merging insecure code. This assures that all\nvulnerabilities have oversight from the appropriate parties.\n\n\n![Merge request approval\npolicy](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097510/Blog/Content%20Images/Blog/Content%20Images/merge_request_approval_policy_aHR0cHM6_1750097509649.png)\n\n\nHackerOne has integrated GitLab into its operations and development\nprocesses in several significant ways, which have led to development process\nimprovements and enhanced scalability and collaboration. These improvements\ninclude faster deployments and cross-team planning.\n\n\n## Key benefits of HackerOne's GitLab integration\n\n\nThe key benefits of using HackerOne and GitLab together include:\n\n\n* **Enhanced security visibility:** Development teams gain immediate\nvisibility into security vulnerabilities without leaving their primary\nworkflow environment. This real-time awareness helps teams prioritize\nsecurity issues alongside feature development.  \n\n* **Streamlined remediation process:** By converting HackerOne reports\ndirectly into GitLab issues, the remediation process becomes part of the\nstandard development cycle. This eliminates context switching between\nplatforms and ensures security fixes are tracked alongside other development\nwork.  \n\n* **Accelerated time to fix:** The integration significantly reduces the\ntime between vulnerability discovery and resolution. With HackerOne\nsubmissions immediately available in GitLab, development teams can begin\nworking on fixes without delay, improving overall security posture.  \n\n* **Improved collaboration:** Security researchers, security teams, and\ndevelopers can communicate more effectively through this integration.\nComments and updates flow between both platforms, creating a collaborative\nenvironment focused on improving security.  \n\n* **Real-world impact:** Organizations implementing the HackerOne and GitLab\nintegration have reported:  \n  * Up to 70% reduction in time from vulnerability discovery to fix  \n  * Improved developer satisfaction by keeping them in their preferred workflow  \n  * Enhanced security visibility across the organization  \n  * More effective allocation of security resources\n\n> To get started today, visit [the integration setup\npage](https://docs.hackerone.com/en/articles/10394699-gitlab-setup) today.\n\n\n## Learn more\n\n\nTo learn more about GitLab and HackerOne, and how we can help enhance your\nsecurity posture, check out the following resources:\n\n* [HackerOne's GitLab Integration\nUsage](https://docs.hackerone.com/en/articles/8571227-gitlab-integration)  \n\n* [HackerOne GitLab Bug Bounty\nProgram](https://hackerone.com/gitlab?type=team)\n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [HackerOne achieves 5x faster deployments with GitLab’s integrated\nsecurity](https://about.gitlab.com/customers/hackerone/)  \n\n* [GitLab Application Security\nDocumentation](https://docs.gitlab.com/ee/user/application_security/)\n",[830,9,232,283,495,851,1804],"bug bounty",{"slug":1806,"featured":6,"template":700},"enhance-application-security-with-gitlab-hackerone","content:en-us:blog:enhance-application-security-with-gitlab-hackerone.yml","Enhance Application Security With Gitlab Hackerone","en-us/blog/enhance-application-security-with-gitlab-hackerone.yml","en-us/blog/enhance-application-security-with-gitlab-hackerone",{"_path":1812,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1813,"content":1819,"config":1824,"_id":1826,"_type":13,"title":1827,"_source":15,"_file":1828,"_stem":1829,"_extension":18},"/en-us/blog/enhance-data-security-with-custom-pii-detection-rulesets",{"title":1814,"description":1815,"ogTitle":1814,"ogDescription":1815,"noIndex":6,"ogImage":1816,"ogUrl":1817,"ogSiteName":685,"ogType":686,"canonicalUrls":1817,"schema":1818},"Strengthen data security with custom PII detection rulesets","This tutorial explains how GitLab's customizable Secret Detection rulesets enhance data security by identifying PII patterns in code repositories. Learn how AI can help.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097701/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%285%29_1iy516k40hwBDChKcUJ2zb_1750097700983.png","https://about.gitlab.com/blog/enhance-data-security-with-custom-pii-detection-rulesets","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Strengthen data security with custom PII detection rulesets\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-04-01\",\n      }",{"title":1814,"description":1815,"authors":1820,"heroImage":1816,"date":1821,"body":1822,"category":830,"tags":1823},[1444],"2025-04-01","Protecting sensitive information is more critical than ever. GitLab's Secret\nDetection feature provides a powerful solution to identify and prevent the\nexposure of sensitive data. This tutorial explores how GitLab Secret\nDetection works, how to create custom rulesets for finding personally\nidentifiable information, and how GitLab Duo Chat can streamline the\ncreation of regex patterns for PII detection.\n\n\n## Understanding GitLab Secret Detection\n\n\n[GitLab Secret\nDetection](https://docs.gitlab.com/user/application_security/secret_detection/)\nis a security scanning feature integrated into the GitLab CI/CD pipeline. It\nautomatically scans your codebase to identify hardcoded secrets,\ncredentials, and other sensitive information that shouldn't be stored in\nyour repository.\n\n\n### Key benefits\n\n\n* **Data breach prevention** detects secrets before they're committed to\nyour repository.  \n\n* **Automated scanning** runs as part of your CI/CD pipeline without manual\nintervention.  \n\n* **Customizable rules** extend detection capabilities with custom\npatterns.  \n\n* **Compliance support** helps meet regulatory requirements like GDPR,\nHIPAA, and the California Privacy Protection Act.\n\n\n## Create custom rulesets for PII detection\n\n\nWhile GitLab's default secret detection covers common secrets like API keys\nand passwords, you may need custom rules to identify specific types of PII\nrelevant to your organization. \n\n\nTo get started, create a new GitLab project and follow the steps below. You\ncan follow along and see usage examples in our [PII Demo\nApplication](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/secret-scanning/pii-data-ruleset).\n\n\n**Step 1: Set up Secret Detection**\n\n\nEnsure Secret Detection is enabled in your `.gitlab-ci.yml` file:\n\n\n```\n\ninclude:\n  - template: Security/Secret-Detection.gitlab-ci.yml\n\nsecret_detection:\n  variables:\n    SECRET_DETECTION_EXCLUDED_PATHS: \"rules,.gitlab,README.md,LICENSE\"\n    SECRET_DETECTION_HISTORIC_SCAN: \"true\"\n```\n\n\n**Step 2: Create a custom ruleset file**\n\n\nCreate the directory and file `rules/pii-data-extenson.toml`, which contains\nthe regex patterns for PII data along with an allowlist of patterns to\nignore. Below are patterns to detect passport numbers (USA), phone numbers\n(USA), and email addresses:\n\n\n```toml\n\n[extend]\n\n# Extends default packaged ruleset, NOTE: do not change the path.\n\npath = \"/gitleaks.toml\"\n\n\n# Patterns to ignore (used for tests)\n\n[allowlist]\n\ndescription = \"allowlist of patterns and paths to ignore in detection\"\n\nregexTarget = \"match\"\n\nregexes = ['''555-555-5555''', '''user@example.com''']\n\npaths = ['''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)''']\n\n\n# US Passport Number (USA)\n\n[[rules]]\n\nid = \"us_passport_detection\"\n\ntitle = \"US Passport Number\"\n\ndescription = \"Detects US passport numbers\"\n\nregex = '''\\b[A-Z]{1,2}[0-9]{6,9}\\b'''\n\nkeywords = [\"passport\"]\n\n\n# Phone Number (USA)\n\n[[rules]]\n\nid = \"us_phone_number_detection_basic\"\n\ntitle = \"US Phone Number\"\n\ndescription = \"Detects US phone numbers in basic format\"\n\nregex = '''\\b\\d{3}-\\d{3}-\\d{4}\\b'''\n\nkeywords = [\"phone\", \"mobile\"]\n\n\n# Email Address\n\n[[rules]]\n\nid = \"email_address\"\n\ntitle = \"Email Address\"\n\ndescription = \"Detects email addresses\"\n\nregex = '''[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}'''\n\nkeywords = [\"email\", \"e-mail\"]\n\n```\n\n**Step 3: Extend Secret Detection with the custom ruleset file**\n\n\nCreate a directory and file `.gitlab/secret-detection-ruleset.toml` in the\nroot of your repository. This file allows you to extend the standard\nconfiguration with the PII rules file, and overwrite the severity of the\ndetected vulnerabilities (default severity is `Critical`).\n\n\n```\n\n# Define the pii rules to add to default configuration\n\n[[secrets.passthrough]]\n\ntype = \"file\"\n\ntarget = \"gitleaks.toml\"\n\nvalue = \"rules/pii-data-extension.toml\"\n\n\n# Overwrite Phone Number (USA) PII Severity\n\n[[secrets.ruleset]]\n\n[secrets.ruleset.identifier]\n\ntype = \"gitleaks_rule_id\"\n\nvalue = \"us_phone_number_detection_basic\"\n\n[secrets.ruleset.override]\n\nseverity = \"Medium\"\n\n\n# Overwrite Email Address PII Severity\n\n[[secrets.ruleset]]\n\n[secrets.ruleset.identifier]\n\ntype = \"gitleaks_rule_id\"\n\nvalue = \"email_address\"\n\n[secrets.ruleset.override]\n\nseverity = \"Low\"\n\n```\n\n\n**Step 4: Commit your changes**\n\n\nNow add the changes in the above steps to your project.\n\n\n```\n\ncd /path/to/your/project\n\ngit add .\n\ngit commit -m \"Add PII data ruleset and Secret Scanning\"\n\ngit push\n\n```\n\n\nOnce the code is committed, Secret Detection will run within the default\nbranch.\n\n\n**Step 5: Test detection of PII data**\n\n\nNow that we have configured the Secret Detection scanner, we should perform\na test to see if the scanner is detecting the new custom patterns. This can\nbe done by creating a merge request, which adds a new file named\n`customer-data.yaml` with the following:\n\n\n```yaml  \n\ncustomers:  \n  test_user:  \n    phone_number: 555-555-555  \n    email: user@example.com  \n  justin_case:  \n    phone_number: 512-123-4567  \n    passport_number: A12345678  \n    email: justin_case@example.com  \n  chris_p_bacon: \n    phone_number: 305-123-4567  \n    passport_number: B09876543  \n    email: chris_p_bacon@example.com  \n```\n\n\nThe scanner should now perform the following:\n\n\n* Ignore the `phone_number` and `email` of `test_user` due to patterns being\nin allowlist  \n\n* Detect six potential vulnerabilities due to the information present for\nboth `justin_case` and `chris_p_bacon`\n  * U.S. passport number severity is set to `Critical` (default)  \n  * U.S. phone number severity is set to `Medium` (override)  \n  * Email address severity is set to `Low` (override)\n  * Data from rules override is added to each vulnerability\n\nOnce the [merge\nrequest](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/secret-scanning/pii-data-ruleset/-/merge_requests/4)\nis submitted, the Secret Detection scanner runs and provides the following\nresults:\n\n\n![Secret Detection finding custom PII data\nMR](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097709/Blog/Content%20Images/Blog/Content%20Images/pii_vulns_aHR0cHM6_1750097709683.png)\n\n\nWhen clicking on a vulnerability, you are presented with detailed\nvulnerability data based on what was configured in your newly set up rules:\n\n\n![Expanded custom PII data\nvulnerability](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097710/Blog/Content%20Images/Blog/Content%20Images/pii_vulns_expanded_aHR0cHM6_1750097709684.png)\n\n\nThis data allows you to determine the validity of the data present and\naddress it accordingly.\n\n\nThere are additional ways to configure custom rulesets. For example, rules\ncan be applied remotely to several projects, avoiding the need to duplicate\nthe rules file. See the [Secret Detection Configuration\ndocumentation](https://docs.gitlab.com/user/application_security/secret_detection/pipeline/configure/)\nfor more information.\n\n\n## Common PII types to consider\n\n\nWhen building your custom ruleset, consider including patterns for:\n\n\n* Social Security Numbers  \n\n* Credit card numbers  \n\n* Driver's license numbers  \n\n* Passport numbers  \n\n* Email addresses  \n\n* Phone numbers  \n\n* IP addresses  \n\n* Physical addresses  \n\n* Medical record numbers\n\n\n## Leveraging GitLab Duo Chat for pattern creation\n\n\nCreating effective regex patterns can be challenging. This is where [GitLab\nDuo Chat](https://docs.gitlab.com/user/gitlab_duo_chat/) comes in handy.\nGitLab Duo Chat is an AI-powered assistant that accelerates development with\ncontextual, conversational AI. Chat is available to any user with an\nassigned GitLab Duo seat.\n\n\n### Using GitLab Duo Chat to generate PII regex patterns\n\n\n1. **Access GitLab Duo Chat:** In the upper right corner, select __GitLab\nDuo Chat__. A drawer opens on the right side of your screen.\n\n\n2. **Prompt the chat:** Enter a prompt in the chat input box and press\n__Enter__ or select __Send__:\n\n\n```\n\nI need a regex pattern to detect Canadian Zip Codes in our codebase for\nsecret detection\n\n```\n\n\n3. **Review and refine:** GitLab Duo Chat will suggest a regex pattern that\nyou can review and refine:\n\n\n```\n\nCanadian postal codes follow the format \"A1A 1A1\" (where A is a letter and 1\nis a digit). Here's a regex pattern that will detect Canadian postal codes\nin your codebase, accounting for both spaced and non-spaced formats:\n\n[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d\n\nThis pattern matches:\n\n* A letter, followed by a digit, followed by a letter\n\n* An optional space or hyphen\n\n* A digit, followed by a letter, followed by a digit\n\nIf you need a more strict pattern that only matches uppercase letters (as\nofficial Canadian postal codes use), you can use:\n\n[A-Z]\\d[A-Z][ -]?\\d[A-Z]\\d\n\nFor implementation in a secret detection tool, you might want to add word\nboundaries to avoid partial matches:\n\n\\b[A-Z]\\d[A-Z][ -]?\\d[A-Z]\\d\\b\n\nWould you like more details about implementing this in a specific secret\ndetection system?\n\n```\n\n\n4. **Test the pattern:** Test the pattern against sample data to ensure it\nworks correctly. You can use sites like\n[https://regex101.com/](https://regex101.com/) to validate patterns before\nyou submit them to your ruleset.\n\n\n5. **Add to your ruleset:** Once satisfied, add the pattern to your\n`rules/pii-data-extension.toml` file and perform some tests to validate.\nThis sample [merge\nrequest](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/secret-scanning/pii-data-ruleset/-/merge_requests/5/diffs)\ncontains a newly added rule for Canadian Zip Codes based on the above.\n\n\nAdditionally, you can use GitLab Duo Chat in:\n\n\n* [The GitLab Web IDE (VS Code in the\ncloud)](https://docs.gitlab.com/user/project/web_ide/)  \n\n* VS Code, with the [GitLab Workflow extension for VS\nCode](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow)  \n\n* JetBrains IDEs, with the [GitLab Duo Plugin for\nJetBrains](https://plugins.jetbrains.com/plugin/22325-gitlab-duo)  \n\n* Visual Studio for Windows, with the [GitLab Extension for Visual\nStudio](https://marketplace.visualstudio.com/items?itemName=GitLab.GitLabExtensionForVisualStudio)\n\n\nIn the future, you’ll be able to leverage [GitLab Duo\nWorkflow](https://docs.gitlab.com/user/duo_workflow/) (currently in private\nbeta) to automatically generate and add these patterns to your code base\ndirectly from your IDE. GitLab Duo Workflow is an AI agent, which transforms\nAI from reactive assistant to autonomous contributor, optimizing your\nsoftware development lifecycle. Learn more about [GitLab Duo\nWorkflow](https://about.gitlab.com/blog/meet-gitlab-duo-workflow-the-future-of-ai-driven-development/).\n\n\n## Best practices for PII detection\n\n\n1. **Start small:** Begin with a few critical PII types and expand\ngradually.  \n\n2. **Test thoroughly:** Test your patterns against sample data to avoid\nfalse positives.  \n\n3. **Update regularly:** Review and update your rulesets as new PII\nrequirements emerge.  \n\n4. **Document patterns:** Maintain documentation for your custom regex\npatterns.  \n\n5. **Balance precision:** Make patterns specific enough to avoid false\npositives but flexible enough to catch variations.  \n\n6. **Implement [Secret Push\nProtection](https://about.gitlab.com/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection/):**\nPrevent PII data from making it into your repository.  \n\n7. **Set up [Merge Request Approval\nPolicies](https://docs.gitlab.com/user/application_security/policies/merge_request_approval_policies/):**\nRequire approval before merging any possible PII data to your repository.\n\n\nOnce you have set up a PII data ruleset to meet your organization's needs,\nremote rulesets can scan for PII data across multiple repositories without\nthe need to duplicate the rules file. Watch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/vjJxQz918WE?si=CRdIEodo3ALxVWXO\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Handling Secret Detection findings\n\n\nWhen GitLab Secret Detection identifies potential PII in your code:\n\n\n1. **Review the finding:** Assess whether it's a legitimate finding or a\nfalse positive.  \n\n2. **Remediate:** Remove the sensitive data and replace it with environment\nvariables or secrets management.  \n\n3. **[Redact text from\nrepository](https://docs.gitlab.com/user/project/merge_requests/revert_changes/#redact-text-from-repository):**\nPermanently delete sensitive or confidential information that was\naccidentally committed, ensuring it's no longer accessible in your\nrepository's history. \n\n4. **Track progress:** Use GitLab's security dashboard to monitor ongoing\ncompliance.\n\n\n## Get started today\n\n\nGitLab Secret Detection, combined with custom PII rulesets, provides a\npowerful defense against inadvertent exposure of sensitive information. By\nleveraging GitLab Duo Chat to create precise regex patterns, teams can\nefficiently implement comprehensive PII detection across their codebase,\nensuring regulatory compliance and protecting user data.\n\n\nRemember that secret detection is just one component of a comprehensive\nsecurity strategy. Combine it with other GitLab security features like\nstatic application security testing, dynamic application security testing,\nand dependency scanning for a more robust security posture.\n\n\nStart implementing these practices today to better protect your users'\npersonal information and maintain the security integrity of your\napplications.\n\n\n> Start [a free trial of GitLab Ultimate and GitLab Duo\n](https://about.gitlab.com/free-trial/)today!\n\n\n## More resources\n\n\nTo learn more about GitLab security and compliance and how we can help\nenhance your AppSec workflows, follow the links below:\n\n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab DevSecOps Platform](https://about.gitlab.com/platform/)  \n\n* [GitLab Duo (AI)](https://about.gitlab.com/gitlab-duo/)  \n\n* [GitLab Application Security\ndocumentation](https://docs.gitlab.com/user/application_security/) \n\n* [Secret Detection\ndocumentation](https://docs.gitlab.com/user/application_security/secret_detection/)\n",[830,694,9,495,720],{"slug":1825,"featured":90,"template":700},"enhance-data-security-with-custom-pii-detection-rulesets","content:en-us:blog:enhance-data-security-with-custom-pii-detection-rulesets.yml","Enhance Data Security With Custom Pii Detection Rulesets","en-us/blog/enhance-data-security-with-custom-pii-detection-rulesets.yml","en-us/blog/enhance-data-security-with-custom-pii-detection-rulesets",{"_path":1831,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1832,"content":1835,"config":1842,"_id":1844,"_type":13,"title":1845,"_source":15,"_file":1846,"_stem":1847,"_extension":18},"/en-us/blog/exact-code-search-find-code-faster-across-repositories",{"noIndex":6,"title":1833,"description":1834},"Exact Code Search: Find code faster across repositories","Discover how this new GitLab feature can find exact matches, use regex patterns, and see contextual results across terabytes of codebases.",{"title":1833,"description":1834,"authors":1836,"heroImage":1838,"date":1839,"body":1840,"category":694,"tags":1841},[1837],"Dmitry Gruzd","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675154/Blog/Hero%20Images/blog-image-template-1800x945__8_.png","2025-06-25","**TL;DR:** What if you could find any line of code across 48 TB of\nrepositories in milliseconds? GitLab's new [Exact Code\nSearch](https://docs.gitlab.com/ee/user/search/exact_code_search.html) makes\nthis possible, delivering pinpoint precision, powerful regex support, and\ncontextual multi-line results that transform how teams work with large\ncodebases.\n\n## Why traditional code search is challenging\n\n\nAnyone who works with code knows the frustration of searching across repositories. Whether you're a developer debugging an issue, a DevOps engineer examining configurations, a security analyst searching for vulnerabilities, a technical writer updating documentation, or a manager reviewing implementation, you know exactly what you need, but traditional search tools often fail you.\n\n\nThese conventional tools return dozens of false positives, lack the context needed to understand results, and slow to a crawl as codebases grow. The result? Valuable time spent hunting for needles in haystacks instead of building, securing, or improving your software.\n\n\nGitLab's code search functionality has historically been backed by Elasticsearch or OpenSearch. While these are excellent for searching issues, merge requests, comments, and other data containing natural language, they weren't specifically designed for code. After [evaluating numerous options](https://gitlab.com/groups/gitlab-org/-/epics/7404), we developed a better solution.\n\n\n## Introducing Exact Code Search: Three game-changing capabilities\n\n\nEnter GitLab's **[Exact Code Search](https://docs.gitlab.com/ee/user/search/exact_code_search.html)**, currently in beta testing and powered by [Zoekt](https://github.com/sourcegraph/zoekt) (pronounced \"zookt\", Dutch for \"search\"). Zoekt is an open-source code search engine originally created by Google and now maintained by Sourcegraph, specifically designed for fast, accurate code search at scale. We've enhanced it with GitLab-specific integrations, enterprise-scale improvements, and seamless permission system integration.\n\n\nThis feature revolutionizes how you find and understand code with three key capabilities:\n\n\n**1. Exact Match mode: Zero false positives**\n\n\nWhen toggled to **Exact Match mode**, the search engine returns only results that match your query exactly as entered, eliminating false positives. This precision is invaluable when:\n\n\n* Searching for specific error messages\n\n* Looking for particular function signatures\n\n* Finding instances of specific variable names\n\n\n**2. Regular Expression mode: Powerful pattern matching**\n\n\nFor complex search needs, Regular Expression mode allows you to craft sophisticated search patterns:\n\n\n* Find functions following specific naming patterns\n\n* Locate variables matching certain constraints\n\n* Identify potential security vulnerabilities using pattern matching\n\n\n**3. Multiple-line matches: See code in context**\n\n\n![Exact Code Search](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750704179/ttjuilkt3v7gtyywnchx.png)\n\n\nInstead of seeing just a single line with your matching term, you get the surrounding context that's crucial for understanding the code. This eliminates the need to click through to files for basic comprehension, significantly accelerating your workflow.\n\n\n## From features to workflows: Real-world use cases and impact\n\n\nLet's see how these capabilities translate to real productivity gains in everyday development scenarios:\n\n\n### Debugging: From error message to root cause in seconds\n\n\nBefore Exact Code Search:\n\nCopy an error message, search, wade through dozens of partial matches in comments and documentation, click through multiple files, and eventually find the actual code.\n\n\nWith Exact Code Search:\n\n\n1. Copy the exact error message\n\n2. Paste it into Exact Code Search with Exact Match mode\n\n3. Instantly find the precise location where the error is thrown, with surrounding context\n\n\n**Impact:** Reduce debugging time from minutes to seconds, eliminating the frustration of false positives.\n\n\n### Code exploration: Master unfamiliar codebases quickly\n\n\nBefore Exact Code Search:\n\nBrowse through directories, make educated guesses about file locations, open dozens of files, and slowly build a mental map of the codebase.\n\n\nWith Exact Code Search:\n\n\n* Search for key methods or classes with Exact Match mode\n\n* Review multiple line matches to understand implementation details\n\n* Use Regular Expression mode to find similar patterns across the codebase\n\n\n**Impact:** Build a mental map of code structure in minutes rather than hours, dramatically accelerating onboarding and cross-team collaboration.\n\n\n### Refactoring with confidence\n\n\nBefore Exact Code Search:\n\nAttempt to find all instances of a method, miss some occurrences, and introduce bugs through incomplete refactoring.\n\n\nWith Exact Code Search:\n\n\n* Use Exact Match mode to find all occurrences of methods or variables\n\n* Review context to understand usage patterns\n\n* Plan your refactoring with complete information about impact\n\n\n**Impact:** Eliminate the \"missed instance\" bugs that often plague refactoring efforts, improving code quality and reducing rework.\n\n\n### Security auditing: Finding vulnerable patterns\n\n\nSecurity teams can:\n\n\n* Create regex patterns matching known vulnerable code\n\n* Search across all repositories in a namespace\n\n* Quickly identify potential security issues with context that helps assess risk\n\n\n**Impact:** Transform security audits from manual, error-prone processes to systematic, comprehensive reviews.\n\n\n### Cross-repository insights\n\n\nSearch across your entire namespace or instance to:\n\n\n* Identify similar implementations across different projects\n\n* Discover opportunities for shared libraries or standardization\n\n\n**Impact:** Break down silos between projects and identify opportunities for code reuse and standardization.\n\n\n## The technical foundation: How Zoekt delivers speed and precision\n\n\nBefore diving into our scale achievements, let's explore what makes Zoekt fundamentally different from traditional search engines — and why it can find exact matches so incredibly fast.\n\n\n### Positional trigrams: The secret to lightning-fast exact matches\n\n\nZoekt's speed comes from its use of **positional trigrams** — a technique that indexes every sequence of three characters along with their exact positions in files. This approach solves one of the biggest pain points developers have had with Elasticsearch-based code search: false positives.\n\n\nHere's how it works:\n\n\n**Traditional full-text search engines** like Elasticsearch tokenize code into words and lose positional information. When you search for `getUserId()`, they might return results containing **user**, **get**, and **Id** scattered throughout a file — leading to those frustrating false positives for GitLab users.\n\n\n**Zoekt's positional trigrams** maintain exact character sequences and their positions. When you search for `getUserId()`, Zoekt looks for the exact trigrams like **get**, **etU**, **tUs**, **Use**, **ser**, **erI**, **rId**, **Id(\", \"d()**, all in the correct sequence and position. This ensures that only exact matches are returned.\n\n\nThe result? Search queries that previously returned hundreds of irrelevant results now return only the precise matches you're looking for. This was [one of our most requested features](https://gitlab.com/gitlab-org/gitlab/-/issues/325234) for good reason - developers were losing significant time sifting through false positives.\n\n\n### Regular expression performance at scale\n\n\nZoekt excels at exact matches and is optimized for regular expression searches. The engine uses sophisticated algorithms to convert regex patterns into efficient trigram queries when possible, maintaining speed even for complex patterns across terabytes of code.\n\n\n## Built for enterprise scale\n\n\nExact Code Search is powerful and built to handle massive scale with impressive performance. This is not just a new UI feature — it's powered by a completely reimagined backend architecture.\n\n\n### Handling terabytes of code with ease\n\n\nOn GitLab.com alone, our Exact Code Search infrastructure indexes and searches over **48 TB** of code data while maintaining lightning-fast response times. This scale represents millions of repositories across thousands of namespaces, all searchable within milliseconds. To put this in perspective: This scale represents more code than the entire Linux kernel, Android, and Chromium projects combined. Yet Exact Code Search can find a specific line across this massive codebase in milliseconds.\n\n\n### Self-registering node architecture\n\n\nOur innovative implementation features:\n\n\n* **Automatic node registration:** Zoekt nodes register themselves with GitLab\n\n* **Dynamic shard assignment:** The system automatically assigns namespaces to nodes\n\n* **Health monitoring:** Nodes that don't check in are automatically marked offline\n\n\nThis self-configuring architecture dramatically simplifies scaling. When more capacity is needed, administrators can simply add more nodes without complex reconfiguration.\n\n\n### Distributed system with intelligent load balancing\n\n\nBehind the scenes, Exact Code Search operates as a distributed system with these key components:\n\n\n* **Specialized search nodes:** Purpose-built servers that handle indexing and searching\n\n* **Smart sharding:** Code is distributed across nodes based on namespaces\n\n* **Automatic load balancing:** The system intelligently distributes work based on capacity\n\n* **High availability:** Multiple replicas ensure continuous operation even if nodes fail\n\n\n*Note: High availability is built into the architecture but not yet fully enabled. See [Issue 514736](https://gitlab.com/gitlab-org/gitlab/-/issues/514736) for updates.*\n\n\n### Seamless security integration\n\n\nExact Code Search automatically integrates with GitLab's permission system:\n\n\n* Search results are filtered based on the user's access rights\n\n* Only code from projects the user has access to is displayed\n\n* Security is built into the core architecture, not added as an afterthought\n\n\n### Optimized performance\n\n\n* **Efficient indexing:** Large repositories are indexed in tens of seconds\n\n* **Fast query execution:** Most searches return results with sub-second response times\n\n* **Streaming results:** The new gRPC-based federated search streams results as they're found\n\n* **Early termination:** Once enough results are collected, the system pauses searching\n\n\n## From library to distributed system: Engineering challenges we solved\n\n\nWhile Zoekt provided the core search technology, it was originally designed as a minimal library for managing `.zoekt` index files - not a distributed database or enterprise-scale service. Here are the key engineering challenges we overcame to make it work at GitLab's scale\"\n\n\n### Challenge 1: Building an orchestration layer\n\n\n**The problem:** Zoekt was designed to work with local index files, not distributed across multiple nodes serving many concurrent users.\n\n\n**Our solution:** We built a comprehensive orchestration layer that:\n\n\n* Creates and manages database models to track nodes, indices, repositories, and tasks\n\n* Implements a self-registering node architecture (inspired by GitLab Runner)\n\n* Handles automatic shard assignment and load balancing across nodes\n\n* Provides bidirectional API communication between GitLab Rails and Zoekt nodes\n\n\n### Challenge 2: Scaling storage and indexing\n\n\n**The problem:** How do you efficiently manage terabytes of index data across multiple nodes while ensuring fast updates?\n\n\n**Our solution:** We implemented:\n\n\n* Intelligent sharding: Namespaces are distributed across nodes based on capacity and load\n\n* Independent replication: Each node independently indexes from [Gitaly](https://gitlab.com/gitlab-org/gitaly) (our Git storage service), eliminating complex synchronization\n\n* Watermark management: Sophisticated storage allocation that prevents nodes from running out of space\n\n* Unified binary architecture: A single `gitlab-zoekt` binary that can operate in both indexer and webserver modes\n\n\n### Challenge 3: Permission Integration\n\n\n**The problem:** Zoekt had no concept of GitLab's complex permission system - users should only see results from projects they can access.\n\n\n**Our solution:** We built native permission filtering directly into the search flow:\n\n\n* Search requests include user permission context\n\n* Results are filtered to include only those the user can access in case permissions change before indexing completes\n\n\n### Challenge 4: Operational simplicity\n\n\n**The problem:** Managing a distributed search system shouldn't require a dedicated team.\n\n\n**Our solution:**\n\n\n* Auto-scaling: Adding capacity is as simple as deploying more nodes - they automatically register and start handling work\n\n* Self-healing: Nodes that don't check in are automatically marked offline and their work redistributed\n\n* Zero-configuration sharding: The system automatically determines optimal shard assignments\n\n\n## Gradual rollout: Minimizing risk at scale\n\n\nRolling out a completely new search backend to millions of users required careful planning. Here's how we minimized customer impact while ensuring reliability:\n\n\n### Phase 1: Controlled testing (gitlab-org group)\n\n\nWe started by enabling Exact Code Search only for the `gitlab-org` group - our own internal repositories. This allowed us to:\n\n\n* Test the system with real production workloads\n\n* Identify and fix performance bottlenecks\n\n* Streamline the deployment process\n\n* Learn from real users' workflows and feedback\n\n\n### Phase 2: Performance validation and optimization\n\n\nBefore expanding, we focused on ensuring the system could handle GitLab.com's scale:\n\n\n* Implemented comprehensive monitoring and alerting\n\n* Validated storage management with real production data growth\n\n\n### Phase 3: Incremental customer expansion\n\n\nWe gradually expanded to customers interested in testing Exact Code Search:\n\n\n* Gathered feedback on performance and user experience\n\n* Refined the search UI based on real user workflows\n\n* Optimized indexing performance (large repositories like `gitlab-org/gitlab` now index in ~10 seconds)\n\n* Refined the architecture based on operational learnings\n\n* Massively increased indexing throughput and improved state transition livecycle\n\n\n### Phase 4: Broad rollout\n\n\nToday, over 99% of Premium and Ultimate licensed groups on GitLab.com have access to Exact Code Search. Users can:\n\n\n* Toggle between regex and exact search modes\n\n* Experience the benefits without any configuration changes\n\n* Fall back to the previous search if needed (though few choose to)\n\n\nRolling this out gradually meant users didn't experience service disruptions, performance degradation, or feature gaps during the transition. We've already received positive feedback from users as they notice their results becoming more relevant and faster.\n\n\n> **For technical deep dive:** Interested in the detailed architecture and implementation? Check out our comprehensive [design document](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/code_search_with_zoekt/) for in-depth technical details about how we built this distributed search system.\n\n\n## Getting started with Exact Code Search\n\n\nGetting started with Exact Code Search is simple because it's already enabled by default for Premium and Ultimate groups on GitLab.com (over 99% of eligible groups currently have access).\n\n\n### Quickstart guide\n\n\n1. Navigate to the Advanced Search in your GitLab project or group\n\n2. Enter your search term in the code tab\n\n3. Toggle between Exact Match and Regular Expression modes\n\n4. Use filters to refine your search\n\n\n### Basic search syntax\n\n\nWhether using Exact Match or Regular Expression mode, you can refine your search with modifiers:\n\n\n| Query Example | What It Does                                             |\n\n| ------------- | -------------------------------------------------------- |\n\n| `file:js`     | Searches only in files containing \"js\" in their name     |\n\n| `foo -bar`    | Finds \"foo\" but excludes results with \"bar\"              |\n\n| `lang:ruby`   | Searches only in Ruby files                              |\n\n| `sym:process` | Finds \"process\" in symbols (methods, classes, variables) |\n\n\n> **Pro Tip:** For the most efficient searches, start specific and then broaden if needed. Using `file:` and `lang:` filters dramatically increases relevance.\n\n\n### Advanced search techniques\n\n\nStack multiple filters for precision:\n\n\n```\n\nis_expected file:rb -file:spec\n\n```\n\n\nThis finds \"is_expected\" in Ruby files that don't have \"spec\" in their name.\n\n\nUse regular expressions for powerful patterns:\n\n\n```\n\ntoken.*=.*[\\\"']\n\n```\n\n\n[Watch this search performed against the GitLab Zoekt repository.](https://gitlab.com/search?search=token.*%3D.*%5B%5C%22'%5D&nav_source=navbar&project_id=46649240&group_id=9970&search_code=true&repository_ref=main&regex=true)\n\n\nThe search helps find hardcoded passwords, which, if not found, can be a security issue.\n\n\nFor more detailed syntax information, check the [Exact Code Search documentation](https://docs.gitlab.com/user/search/exact_code_search/#syntax).\n\n\n## Availability and deployment\n\n\n### Current availability\n\n\nExact Code Search is currently in Beta for GitLab.com users with Premium and Ultimate licenses:\n\n\n* Available for over 99% of licensed groups\n\n* Search in the UI automatically uses Zoekt when available, Exact Code Search in Search API is behind a feature flag\n\n\n### Self-managed deployment options\n\n\nFor self-managed instances, we offer several deployment methods:\n\n\n* Kubernetes/Helm: Our most well-supported method, using our [`gitlab-zoekt` Helm chart](https://gitlab.com/gitlab-org/cloud-native/charts/gitlab-zoekt)\n\n* Other deployment options: We're working on streamlining deployment for Omnibus and other installation methods\n\n\nSystem requirements depend on your codebase size, but the architecture is designed to scale horizontally and/or vertically as your needs grow.\n\n\n## What's coming next\n\n\nWhile Exact Code Search is already powerful, we're continuously improving it:\n\n\n* **Scale optimizations** to support instances with hundreds of thousands of repositories\n\n* **Improved self-managed deployment** options, including streamlined Omnibus support\n\n* **Full high availability support** with automatic failover and load balancing\n\n\nStay tuned for updates as we move from Beta to General Availability.\n\n\n## Transform how you work with code\n\n\nGitLab's Exact Code Search represents a fundamental rethinking of code discovery. By delivering exact matches, powerful regex support, and contextual results, it solves the most frustrating aspects of code search:\n\n\n* No more wasting time with irrelevant results\n\n* No more missing important matches\n\n* No more clicking through files just to understand basic context\n\n* No more performance issues as codebases grow\n\n\nThe impact extends beyond individual productivity:\n\n\n* **Teams collaborate better** with easy code referencing\n\n* **Knowledge sharing accelerates** when patterns are discoverable\n\n* **Onboarding becomes faster** with quick codebase comprehension\n\n* **Security improves** with effective pattern auditing\n\n* **Technical debt reduction** becomes more feasible\n\n\nExact Code Search isn't just a feature, it's a better way to understand and work with code. Stop searching and start finding.\n\n\n**We'd love to hear from you!** Share your experiences, questions, or feedback about Exact Code Search in our [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/420920). Your input helps us prioritize improvements and new features.\n\n\n> #### Ready to experience smarter code search? Learn more in our [documentation](https://docs.gitlab.com/ee/user/search/exact_code_search.html) or try it now by performing a search in your Premium or Ultimate licensed namespaces or projects. Not a GitLab user yet? Try [a free trial of GitLab Ultimate with Duo](https://about.gitlab.com/free-trial/)!\n",[694,9,743],{"featured":6,"template":700,"slug":1843},"exact-code-search-find-code-faster-across-repositories","content:en-us:blog:exact-code-search-find-code-faster-across-repositories.yml","Exact Code Search Find Code Faster Across Repositories","en-us/blog/exact-code-search-find-code-faster-across-repositories.yml","en-us/blog/exact-code-search-find-code-faster-across-repositories",{"_path":1849,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1850,"content":1856,"config":1861,"_id":1863,"_type":13,"title":1864,"_source":15,"_file":1865,"_stem":1866,"_extension":18},"/en-us/blog/exporting-vulnerability-reports-to-html-pdf-jira",{"title":1851,"description":1852,"ogTitle":1851,"ogDescription":1852,"noIndex":6,"ogImage":1853,"ogUrl":1854,"ogSiteName":685,"ogType":686,"canonicalUrls":1854,"schema":1855},"How to export vulnerability reports to HTML/PDF and Jira","With GitLab's API, it's easy to query vulnerability info and send the report details elsewhere, such as a PDF file or a Jira project.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662877/Blog/Hero%20Images/security-cover-new.png","https://about.gitlab.com/blog/exporting-vulnerability-reports-to-html-pdf-jira","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to export vulnerability reports to HTML/PDF and Jira\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Siddharth Mathur\"}],\n        \"datePublished\": \"2023-09-14\",\n      }",{"title":1851,"description":1852,"authors":1857,"heroImage":1853,"date":1858,"body":1859,"category":785,"tags":1860},[973],"2023-09-14","\nGitLab's [Vulnerability Report](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/) makes it easy to triage security scan results without ever having to leave the platform. You can manage your code, run security scans against it, and fix vulnerabilities all in one place. That being said, some teams prefer to manage their vulnerabilities in a separate tool like Jira. They may also need to present the vulnerability report to leadership in a digestible format.\n\nOut of the box, GitLab's Vulnerability Report can be [exported to CSV](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/#export-vulnerability-details) with a single click, for easy analysis in other tools. In some cases though, a simple PDF of the report is all that's needed. \n\nWith [GitLab's API](https://docs.gitlab.com/ee/api/graphql/reference/index.html#queryvulnerabilities), it's easy to query vulnerability info and send the report details elsewhere, such as a PDF file or a Jira project. In this blog, we'll show you how to export to HTML/PDF and Jira. **Note that the scripts used in this tutorial are provided for educational purposes and they are not supported by GitLab.**\n\n## Exporting to HTML/PDF\nTo export your vulnerability reports to HTML or PDF, head to the [Custom Vulnerability Reporting](https://gitlab.com/jwagner-demo/vandelay-industries/engineering/custom-vulnerability-reporting) project. \n\n![Project overview](https://about.gitlab.com/images/blogimages/2023-07-27-exporting-vulnerability-reports-to-html-pdf-and-jira/project_overview.png)\n\n\nThis project contains a script that queries a project's vulnerability report, and then generates an HTML file from that data. The pipeline configured in the project runs this script and converts the HTML file to PDF as well.\n\nTo use the exporter, first [fork the project](https://gitlab.com/jwagner-demo/vandelay-industries/engineering/custom-vulnerability-reporting/-/forks/new) or [import it into a new project](https://gitlab.com/projects/new#import_project) (select “Repository by URL” and paste the git URL of the original project).\n\n![Project import](https://about.gitlab.com/images/blogimages/2023-07-27-exporting-vulnerability-reports-to-html-pdf-and-jira/project_import.png)\n\n\nSet the CI/CD variables as described in the readme. You'll need the following from GitLab:\n- GitLab project/personal access token with permissions to access vulnerability info (read_api scope)\n- GitLab GraphQL API URL (for SaaS this is https://gitlab.com/api/graphql)\n- GitLab project path (e.g. smathur/custom-vulnerability-reporting)\n\nAfter you've set the required CI/CD variables, manually run a pipeline from your project's Pipelines page. Once the pipeline is complete, you'll see your file export by going to the “build_report” (for HTML) or “pdf_conversion” job and selecting “Download” or “Browse” on the sidebar under \"Job artifacts.\" And there you have it! A shareable, easy-to-read export of your project's vulnerabilities.\n\n![PDF export](https://about.gitlab.com/images/blogimages/2023-07-27-exporting-vulnerability-reports-to-html-pdf-and-jira/pdf_export.png)\n\n\n## Exporting vulnerability info to Jira\nGitLab lets you create Jira tickets from vulnerabilities through the UI using our [Jira integration](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#create-a-jira-issue-for-a-vulnerability). While you can do this individually for vulnerabilities that need actioning, sometimes teams need to bulk-create Jira tickets for all their vulnerabilities. We can leverage GitLab and Jira's APIs to achieve this.\n\nTo get started, head to the [External Vulnerability Tracking](https://gitlab.com/smathur/external-vulnerability-tracking) project. This script fetches vulnerabilities in the same way as the script above, but it uses the Jira API to create a ticket for each vulnerability. Each ticket's description is also populated with details from GitLab's vulnerability report.\n\nTo use the exporter, simply [fork the project](https://gitlab.com/smathur/external-vulnerability-tracking/-/forks/new) or [import it into a new project](https://gitlab.com/projects/new#import_project) (select “Repository by URL” and paste the git URL of the original project), and set the CI/CD variables as described in the readme. You'll need the following from GitLab:\n- GitLab project/personal access token with permissions to access vulnerability info (read_api scope)\n- GitLab GraphQL API URL (for SaaS this is https://gitlab.com/api/graphql)\n- GitLab project path (e.g. smathur/external-vulnerability-tracking)\n\nYou will also need the following from Jira:\n- Jira [personal access token](https://id.atlassian.com/manage-profile/security/api-tokens)\n- Jira API issue endpoint URL (for SaaS this is https://ORG_NAME.atlassian.net/rest/api/latest/issue/)\n- Jira user email ID\n- Jira project key where you want to create vulnerability tickets (e.g. ABC)\n\nOnce you have set your CI/CD variables as described in the project readme, simply run a pipeline from your project's Pipelines page, and watch as your tickets get created in Jira!\n\nIf you run the pipeline again in the future, the script will run a search query against your Jira project to prevent duplicate tickets from being created. It will create tickets for new vulnerabilities that aren't already in Jira.\n\n![Jira export](https://about.gitlab.com/images/blogimages/2023-07-27-exporting-vulnerability-reports-to-html-pdf-and-jira/jira_export.png)\n\n\n## References\n- [GitLab Vulnerability API](https://docs.gitlab.com/ee/api/graphql/reference/index.html#queryvulnerabilities)\n- [Custom Vulnerability Reporting project](https://gitlab.com/jwagner-demo/vandelay-industries/engineering/custom-vulnerability-reporting)\n- [External Vulnerability Tracking project](https://gitlab.com/smathur/external-vulnerability-tracking)\n- [Jira REST API examples](https://developer.atlassian.com/server/jira/platform/jira-rest-api-examples/)\n\n",[9,765,830,810],{"slug":1862,"featured":6,"template":700},"exporting-vulnerability-reports-to-html-pdf-jira","content:en-us:blog:exporting-vulnerability-reports-to-html-pdf-jira.yml","Exporting Vulnerability Reports To Html Pdf Jira","en-us/blog/exporting-vulnerability-reports-to-html-pdf-jira.yml","en-us/blog/exporting-vulnerability-reports-to-html-pdf-jira",{"_path":1868,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1869,"content":1872,"config":1878,"_id":1880,"_type":13,"title":1881,"_source":15,"_file":1882,"_stem":1883,"_extension":18},"/en-us/blog/fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab",{"noIndex":6,"title":1870,"description":1871},"Fast and secure AI agent deployment to Google Cloud with GitLab","Follow this step-by-step guide, complete with a demo application, to learn how to use agentic AI, along with GitLab's native integrations and CI/CD components.",{"title":1870,"description":1871,"authors":1873,"heroImage":1875,"date":828,"body":1876,"category":718,"tags":1877},[1874],"Regnard Raquedan","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749670563/Blog/Hero%20Images/cloudcomputing.jpg","[Agentic AI](https://about.gitlab.com/topics/agentic-ai/) is transforming\nhow we build intelligent applications, but deploying AI agents securely and\nefficiently can be challenging. In this tutorial, you'll learn how to deploy\nan AI agent built with Google's Agent Development Kit\n([ADK](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-development-kit/quickstart))\nto Cloud Run using [GitLab's native\nintegrations](https://cloud.google.com/blog/topics/partners/understand-the-google-cloud-gitlab-integration)\nand [CI/CD components](https://docs.gitlab.com/ci/components/).\n\n\n## What are AI agents and why do they matter?\n\n\nAgentic AI represents a significant evolution in artificial intelligence. Unlike traditional generative AI tools that require constant human direction, AI agents leverage advanced language models and natural language processing to take independent action. These systems can understand requests, make decisions, and execute multistep plans to achieve goals autonomously.\n\n\nThis tutorial uses Google's ADK, a flexible and modular framework for developing and deploying AI agents. While optimized for Gemini and the Google ecosystem, ADK is model-agnostic, deployment-agnostic, and built for compatibility with other frameworks.\n\n\n## Our demo application: Canada City Advisor\n\n\nTo demonstrate the deployment process, we'll work with a practical example: the Canada City Advisor. This AI agent helps users find their ideal Canadian city based on their preferences and constraints.\n\n\nHere's how it works:\n\n\n* Users input their budget requirements and lifestyle preferences.  \n\n* The root agent coordinates two sub-agents:  \n\n  * A budget analyzer agent that evaluates financial constraints. This draws data obtained from the Canada Mortgage and Housing Corporation.  \n  * A lifestyle preferences agent that matches cities to user needs. This includes a weather service that uses [Open-Meteo](https://open-meteo.com/) to get the proper city information.  \n* The system generates personalized city recommendations\n\n\nThis multi-agent architecture showcases the power of agentic AI - different specialized agents working together to solve a complex problem. The sub-agents are only invoked when the root agent determines that budget and lifestyle analysis are needed.\n\n\n![Multi-agent architecture to develop demo application with agentic AI](https://res.cloudinary.com/about-gitlab-com/image/upload/v1751576568/obgxpxvlnxtzifddrrz1.png)\n\n\n## Prerequisites\n\n\nBefore we begin, ensure you have:\n\n\n* A Google Cloud project with the following APIs enabled:  \n\n  * Cloud Run API  \n  * Artifact Registry API  \n  * Vertex AI API  \n* A GitLab project for your source code  \n\n* Appropriate permissions in both GitLab and Google Cloud\n\n\n**Step 1: Set up IAM integration with Workload Identity Federation**\n\n\nThe first step establishes secure, keyless authentication between GitLab and Google Cloud using [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation). This eliminates the need for service account keys and improves security.\n\n\nIn your GitLab project:\n\n\n1. Navigate to **Settings > Integrations > Google Cloud IAM.**  \n\n2. Provide the following information:  \n\n   * **Project ID**: Your Google Cloud project ID  \n   * **Project Number**: Found in your Google Cloud console  \n   * **Pool ID**: A unique identifier for your workload identity pool  \n   * **Provider ID**: A unique identifier for your identity provider\n\nGitLab will generate a script for you. Copy this script and run it in your Google Cloud Shell to create the Workload Identity Federation.\n\n\n**Step 2: Configure Google Artifact Registry integration**\n\n\nNext, we'll set up the connection to Google Artifact Registry where our container images will be stored.\n\n\n1. In GitLab, go to **Settings > Integrations > Google Artifact Registry.**  \n\n2. Enter:  \n\n   * **Google Cloud Project ID**: Same as in Step 1  \n   * **Repository Name**: Name of an existing Artifact Registry repository  \n   * **Location**: The region where your repository is located\n\n**Important**: The repository must already exist in Artifact Registry. GitLab won't create a new one for you in this context.\n\n\nGitLab will generate commands to set up the necessary permissions. Run these in Google Cloud Shell.\n\n\nAdditionally, add these roles to your service principal for Cloud Run deployment:\n\n\n* `roles/run.admin`  \n\n* `roles/iam.serviceAccountUser`  \n\n* `roles/cloudbuild.builds.editor`\n\n\nYou can add these roles using the following gcloud commands:\n\n\n```shell\n\nGCP_PROJECT_ID=\"\u003Cyour-project-id>\" #replace\n\nGCP_PROJECT_NUMBER=\"\u003Cyour-project-number>\" #replace\n\nGCP_WORKLOAD_IDENTITY_POOL=\"\u003Cyour-pool-id>\" #replace\n\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/run.admin'\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/iam.serviceAccountUser'\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/cloudbuild.builds.editor'\n```\n\n\n**Step 3: Create the CI/CD pipeline**\n\n\nNow for the exciting part – let's build our deployment pipeline! GitLab's CI/CD components make this remarkably simple.\n\n\nCreate a `.gitlab-ci.yml` file in your project root:\n\n\n```unset\n\nstages:\n  - build\n  - test\n  - upload\n  - deploy\n\nvariables:\n  GITLAB_IMAGE: $CI_REGISTRY_IMAGE/main:$CI_COMMIT_SHORT_SHA\n  AR_IMAGE: $GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_LOCATION-docker.pkg.dev/$GOOGLE_ARTIFACT_REGISTRY_PROJECT_ID/$GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_NAME/main:$CI_COMMIT_SHORT_SHA\n\nbuild:\n  image: docker:24.0.5\n  stage: build\n  services:\n    - docker:24.0.5-dind\n  before_script:\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n  script:\n    - docker build -t $GITLAB_IMAGE .\n    - docker push $GITLAB_IMAGE\n\ninclude:\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml\n  - template: Jobs/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml\n  - template: Jobs/Secret-Detection.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml\n  - component: gitlab.com/google-gitlab-components/artifact-registry/upload-artifact-registry@main\n    inputs:\n      stage: upload\n      source: $GITLAB_IMAGE\n      target: $AR_IMAGE\n  - component: gitlab.com/google-gitlab-components/cloud-run/deploy-cloud-run@main\n    inputs:\n      stage: deploy\n      project_id: \"\u003Cyour-project-id>\" #replace\n      service: \"canadian-city\"\n      region: \"us-central1\"\n      image: $AR_IMAGE\n```\n\n\nThe pipeline consists of four stages:\n\n\n1. **Build**: Creates the Docker container with your AI agent  \n\n2. **Test**: Runs security scans (container scanning, dependency scanning, SAST)  \n\n3. **Upload**: Pushes the container to Artifact Registry  \n\n4. **Deploy**: Deploys to Cloud Run\n\n\nThe great thing about using [GitLab's CI/CD components](https://docs.gitlab.com/ci/components/) is that you only need to provide a few parameters - the components handle all the complex authentication and deployment logic.\n\n\n**Step 4: Deploy and test**\n\n\nWith everything configured, it's time to deploy:\n\n\n1. Commit your code and `.gitlab-ci.yml` to your GitLab repository.  \n\n2. The pipeline will automatically trigger.  \n\n3. Monitor the pipeline progress in GitLab's CI/CD interface.  \n\n4. Once complete, find your Cloud Run URL in the Google Cloud Console.\n\n\nYou'll see each stage execute:\n\n\n* Build stage creates your container.  \n\n* Test stage runs comprehensive security scans.  \n\n* Upload stage pushes to Artifact Registry.  \n\n* Deploy stage creates or updates your Cloud Run service.\n\n\n## Security benefits\n\n\nThis approach provides several security advantages:\n\n\n* **No long-lived credentials:** Workload Identity Federation eliminates service account keys.  \n\n* **Automated security scanning:** Every deployment is scanned for vulnerabilities.  \n\n* **Audit trail:** Complete visibility of who deployed what and when.  \n\n* **Principle of least privilege:** Fine-grained IAM roles limit access.\n\n\n## Summary\n\nBy combining GitLab's security features with Google Cloud's powerful AI and serverless platforms, you can deploy AI agents that are both secure and scalable. The integration between GitLab and Google Cloud eliminates much of the complexity traditionally associated with such deployments.\n\n> Use this tutorial's [complete code\nexample](https://gitlab.com/gitlab-partners-public/google-cloud/demos/ai-agent-deployment)\nto get started now. Not a GitLab customer yet? Explore the DevSecOps platform with [a free trial](https://about.gitlab.com/free-trial/).\n",[720,1264,9],{"featured":6,"template":700,"slug":1879},"fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab","content:en-us:blog:fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab.yml","Fast And Secure Ai Agent Deployment To Google Cloud With Gitlab","en-us/blog/fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab.yml","en-us/blog/fast-and-secure-ai-agent-deployment-to-google-cloud-with-gitlab",{"_path":1885,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1886,"content":1892,"config":1899,"_id":1901,"_type":13,"title":1902,"_source":15,"_file":1903,"_stem":1904,"_extension":18},"/en-us/blog/fast-python-flask-server-deployment-with-gitlab-google-cloud",{"title":1887,"description":1888,"ogTitle":1887,"ogDescription":1888,"noIndex":6,"ogImage":1889,"ogUrl":1890,"ogSiteName":685,"ogType":686,"canonicalUrls":1890,"schema":1891},"Fast Python Flask server deployment with GitLab + Google Cloud","This tutorial shows how to use GitLab’s Google Cloud integration to deploy a Python Flask server in less than 10 minutes, helping developers become more independent and efficient.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098427/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945_fJKX41PJHKCfSOWw4xQxm_1750098427691.png","https://about.gitlab.com/blog/fast-python-flask-server-deployment-with-gitlab-google-cloud","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Fast Python Flask server deployment with GitLab + Google Cloud\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Noah Ing\"},{\"@type\":\"Person\",\"name\":\"Jerez Solis\"}],\n        \"datePublished\": \"2024-11-04\",\n      }",{"title":1887,"description":1888,"authors":1893,"heroImage":1889,"date":1895,"body":1896,"category":785,"tags":1897},[847,1894],"Jerez Solis","2024-11-04","Deploying an application to the cloud often requires assistance from\nproduction or DevOps engineers. GitLab's Google Cloud integration empowers\ndevelopers to handle deployments independently. In this tutorial, you'll\nlearn how to deploy a Python Flask server to Google Cloud in less than 10\nminutes. Whether you’re a solo developer or part of a large team, this setup\nallows you to deploy applications efficiently.\n\n\nYou'll learn how to:\n\n\n- Create a new project in GitLab\n\n- Create a Flask server utilizing `main.py`\n\n- Utilize the Google Cloud integration to create a Service account\n\n- Utilize the Google Cloud integration to create Cloud Run via a merge\nrequest\n\n- Access your newly deployed Flask server\n\n- Clean up your environment\n\n\n## Prerequisites:\n\n- Owner access on a Google Cloud Platform project\n\n- Working knowledge of Python\n\n- Working knowledge of GitLab CI\n\n- 10 minutes\n\n\n## Step-by-step Python Flask server deployment to Google Cloud\n\n\n**1. Create a new project in GitLab.**\n\n\nWe decided to call our project \"python-flask-cloud-run\" for simplicity.\n\n\n![python flask server - create a new project in\nGitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098438036.png)\n\n\n**2. Create a flask server utilizing main.py demo.**\n\n\nFind the `main.py` demo here:\n[https://gitlab.com/demos/applications/python-flask-cloud-run](https://gitlab.com/demos/applications/python-flask-cloud-run).\n\n\n```python\n\nimport os\n\n\nfrom flask import Flask\n\n\napp = Flask(__name__)\n\n\n@app.route(\"/\")\n\ndef hello_world():\n    \"\"\"Example Hello World route.\"\"\"\n    name = os.environ.get(\"NAME\", \"World\")\n    return f\"Hello {name}!\"\n\nif __name__ == \"__main__\":\n    app.run(debug=True, host=\"0.0.0.0\", port=int(os.environ.get(\"PORT\", 8080)))\n```\n\n\n**3. Create a `requirements.txt` with the following dependencies.**\n\n\n```\n\nFlask==3.0.3\n\ngunicorn==22.0.0\n\nWerkzeug==3.0.3\n\n```\n\n\n**4. Utilizing the Google Cloud integration, create a Service account.**\n\n\nNavigate to **Operate > Google Cloud > Create Service account**.\n\n\n![python flask server - create service\naccount](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098438037.png)\n\n\n**5. Also configure the region you would like the Cloud Run instance to\ndeploy to.**\n\n\n![python flask server - configure the\nregion](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098438038.png)\n\n\n**6. Utilizing the Google Cloud integration, configure Cloud Run via merge\nrequest.**\n\n\n![python flask server -\ndeployments](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098438041.png)\n\n\n**7. This will open a merge request. Immediately merge this merge request.**\n\n\n![python flask server - enable deployments to Cloud\nRun](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098438043.png)\n\n\n**Note:** `GCP_PROJECT_ID`, `GCP_REGION`, `GCP_SERVICE_ACCOUNT`,\n`GCP_SERVICE_ACCOUNT_KEY` will all be automatically populated from the\nprevious steps.\n\n\n![python flask server -\nvariables](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098438044.png)\n\n\n**8. Voila! Check your pipeline and you will see you have successfully\ndeployed to Google Cloud Run utilizing GitLab CI.**\n\n\n![python flask server - update\ndockerfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098438045.png)\n\n\n\u003Cbr>\u003C/br>\n\n\n![python flask server -\ndockerfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098438046.png)\n\n\n**9. Click the Service URL to view your newly deployed Flask server.**\n\n\nNavigate to **Operate > Environments** to see a list of deployments for your\nenvironments.\n\n\n![python flask server - deployments\nlist](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098438047.png)\n\n\nBy clicking on the environment called **main**, you’ll be able to view a\ncomplete list of deployments specific to that environment.\n\n\n![python flask server - main job\nlisting](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098438/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098438048.png)\n\n\n## Next steps\n\n\nTo get started with developing your Flask application, try adding another\nendpoint. For instance, in your `main.py` file, you can add a **/bye**\nendpoint as shown below:\n\n\n```\n\n@app.route(\"/\")\n\ndef hello_world():\n    \"\"\"Example Hello World route.\"\"\"\n    name = os.environ.get(\"NAME\", \"World\")\n    return f\"Hello {name}!\"\n\n```\n\n\nPush the changes to the repo, and watch the `deploy-to-cloud-run` job deploy\nthe updates. Once it’s complete, go back to the Service URL and navigate to\nthe **/bye** endpoint to see the new functionality in action.\n\n\n## Clean up\n\n\nTo prevent incurring charges on your Google Cloud account for the resources\nused in this tutorial, you can either delete the specific resources or\ndelete the entire Google Cloud project. For detailed instructions, refer to\nthe [cleanup\nguide](https://docs.gitlab.com/ee/tutorials/create_and_deploy_web_service_with_google_cloud_run_component/#clean-up).\n\n\n> For more DevSecOps capabilities, [start a free trial of GitLab\nUltimate and GitLab\nDuo](https://gitlab.com/-/trials/new?glm_content=default-saas-trial&glm_source=about.gitlab.com/blog/%2F).\n",[9,1898,1264,1016],"cloud native",{"slug":1900,"featured":90,"template":700},"fast-python-flask-server-deployment-with-gitlab-google-cloud","content:en-us:blog:fast-python-flask-server-deployment-with-gitlab-google-cloud.yml","Fast Python Flask Server Deployment With Gitlab Google Cloud","en-us/blog/fast-python-flask-server-deployment-with-gitlab-google-cloud.yml","en-us/blog/fast-python-flask-server-deployment-with-gitlab-google-cloud",{"_path":1906,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1907,"content":1913,"config":1919,"_id":1921,"_type":13,"title":1922,"_source":15,"_file":1923,"_stem":1924,"_extension":18},"/en-us/blog/from-code-to-production-a-guide-to-continuous-deployment-with-gitlab",{"title":1908,"description":1909,"ogTitle":1908,"ogDescription":1909,"noIndex":6,"ogImage":1910,"ogUrl":1911,"ogSiteName":685,"ogType":686,"canonicalUrls":1911,"schema":1912},"From code to production: A guide to continuous deployment with GitLab","Learn how to get started building a robust continuous deployment pipeline in GitLab. Follow these step-by-step instructions, practical examples, and best practices.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659478/Blog/Hero%20Images/REFERENCE_-_Use_this_page_as_a_reference_for_thumbnail_sizes.png","https://about.gitlab.com/blog/from-code-to-production-a-guide-to-continuous-deployment-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"From code to production: A guide to continuous deployment with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Benjamin Skierlak\"},{\"@type\":\"Person\",\"name\":\"James Wormwell\"}],\n        \"datePublished\": \"2025-01-28\",\n      }",{"title":1908,"description":1909,"authors":1914,"heroImage":1910,"date":1544,"body":1917,"category":694,"tags":1918},[1915,1916],"Benjamin Skierlak","James Wormwell","Continuous deployment is a game-changing practice that enables teams to\ndeliver value faster, with higher confidence. However, diving into advanced\ndeployment workflows — such as GitOps, container orchestration with\nKubernetes, or dynamic environments — can be intimidating for teams just\nstarting out.\n\n\nAt GitLab, we're committed to making delivery seamless and scalable. By\nenabling teams to focus on the fundamentals, we empower them to build a\nstrong foundation that supports growth into more complex strategies over\ntime. This guide provides essential steps to begin implementing continuous\ndeployment with GitLab, laying the foundation for your long-term success.\n\n\n## Start with a workflow plan\n\n\nBefore diving into the technical implementation, take time to map out your\ndeployment workflow. Success lies in careful planning and a methodical\napproach.\n\n\n### Artifact management strategy\n\n\nIn the context of continuous deployment, artifacts are the packaged outputs\nof your build process that need to be stored, versioned, and deployed. These\ncould be:\n\n\n- container images for your applications\n\n- packages\n\n- compiled binaries or executables\n\n- libraries\n\n- configuration files\n\n- documentation packages\n\n- other artifacts\n\n\nEach type of artifact plays a specific role in your deployment process. For\nexample, a typical web application might generate:\n\n\n- a container image for the backend service\n\n- a ZIP archive of compiled frontend assets\n\n- SQL files for database changes\n\n- environment-specific configuration files\n\n\nManaging these artifacts effectively is crucial for successful deployments.\nHere's how to approach artifact management.\n\n\n#### Artifacts and releases versioning strategies\n\n\nA best practice to get you started with a clean structure is to establish a\nclear versioning strategy for your artifacts. When creating releases:\n\n\n- Use semantic versioning (major.minor.patch) for release tags\n  - Example: `myapp:1.2.3` for a stable release\n  - Major version changes (2.0.0) for breaking changes\n  - Minor version changes (1.3.0) for new features\n  - Patch version changes (1.2.4) for bug fixes\n- Maintain a 'latest' tag for the most recent stable version\n  - Example: `myapp:latest` for automated deployments\n- Include commit SHA for precise version tracking\n  - Example: `myapp:1.2.3-abc123f` for debugging\n- Consider branch-based tags for development environments\n  - Example: `myapp:feature-user-auth` for feature testing\n\n#### Build artifacts retention\n\n\nImplement defined retention rules:\n\n\n- Set explicit expiration timeframes for temporary artifacts\n\n- Define which artifacts need permanent retention\n\n- Configure cleanup policies to manage storage\n\n\n#### Registry access and authentication\n\n\nSecure your artifacts with proper access controls:\n\n\n- Implement Personal Access Tokens for developer access\n\n- Configure CI/CD variables for pipeline authentication\n\n- Set up proper access scopes\n\n\n### Environment strategy\n\n\nConsider your environments early, as they shape your entire deployment\npipeline:\n\n\n- Development, staging, and production environment configurations\n\n- Environment-specific variables and secrets\n\n- Access controls and protection rules\n\n- Deployment tracking and monitoring approach\n\n\n### Deployment targets\n\n\nBe intentional as to where and how you'll deploy, these decisions matter and\nthe benefits and drawbacks of each should be consider:\n\n\n- Infrastructure requirements (VMs, containers, cloud services)\n\n- Network access and security configurations\n\n- Authentication mechanisms (SSH keys, access tokens)\n\n- Resource allocation and scaling considerations\n\n\nWith our strategy defined and foundational decisions made, we can now\ntranslate these plans into a working pipeline. We'll build a practical\nexample that demonstrates these concepts, starting with a simple application\nand progressively adding deployment capabilities.\n\n\n## Implementing your CD pipeline\n\n\n### A step-by-step example\n\n\nLet's walk through implementing a basic continuous deployment pipeline for a\nweb application. We'll use a simple HTML application as an example, but\nthese principles apply to any type of application. We’re also going to\ndeploy our application as a Docker image on a simple virtual machine. This\nwill allow us to lean on a curated image with minimum dependencies, and to\nensure no environment specific requirements are unintentionally brought in.\nBy working on a virtual machine, we won’t be leveraging GitLab’s native\nintegrations, allowing us to work on an easier but less scalable setup to\nbegin with.\n\n\n#### Prerequisites\n\n\nIn this example, we’ll aim to containerize an application that we’ll run on\na virtual machine hosted on a cloud provider. We’ll also test this\napplication locally on our machine. This list of prerequisites is only\nneeded for this scenario.\n\n\n##### Virtual machine setup\n\n\n- Provision a VM in your preferred cloud provider (e.g., GCP, AWS, Azure)\n\n- Configure network rules to allow access on ports 22, 80, and 443\n\n- Record the machine's public IP address for deployment\n\n\n##### Set up SSH authentication:\n\n\n- Generate a public/private key pair for the machine\n\n- In GitLab, go to **Settings > CI/CD > Variables**\n\n- Create a variable called `GITLAB_KEY`\n\n- Set Type to \"File\" (required for SSH authentication)\n\n- Paste the private key in the Value field\n\n- Define a USER variable, this is the user logging in and running the\nscripts on your VM\n\n\n##### Configure deployment variables\n\n\n- Create variables for your deployment targets:\n  - `STAGING_TARGET`: Your staging server IP/domain\n  - `PRODUCTION_TARGET`: Your production server IP/domain\n\n##### Local development setup\n\n\n- Install Docker on your local machine for testing deployments\n\n\n##### GitLab Container Registry access\n\n\n- Locate your registry path:\n  - Navigate to **Deploy > Container Registry**\n  - Copy the registry path (e.g., registry.gitlab.com/group/project)\n- Set up authentication:\n  - Go to **Settings > Access Tokens**\n  - Create a new token with registry access\n  - Token expiration: Maximum 1 year\n  - Save the token securely\n- Configure local registry access:\n\n\n```\n\ndocker login registry.gitlab.com\n\n# The username if you are using a PAT is gitlab-ci-token\n\n# Password: your-access-token\n\n```\n\n\n#### 1. Create your application\n\n\nStart with a basic web application. For our example, we're using a simple\nHTML page:\n\n\n```\n\n\u003C!-- index.html -->\n\n\u003Chtml>\n  \u003Chead>\n    \u003Cstyle>\n      body {\n        background-color: #171321; /* GitLab dark */\n      }\n    \u003C/style>\n  \u003C/head>\n  \u003Cbody>\n    \u003C!-- Your content here -->\n  \u003C/body>\n\u003C/html>\n\n```\n\n\n#### 2. Containerize your application\n\n\nCreate a Dockerfile to package your application:\n\n\n```\n\nFROM nginx:1.26.2\n\nCOPY index.html /usr/share/nginx/html/index.html\n\n```\n\n\nThis Dockerfile:\n\n\n- Uses nginx as a base image for serving web content\n\n- Copies your HTML file to the correct location in the nginx directory\nstructure\n\n\n#### 3. Set up your CI/CD pipeline\n\n\nCreate a `.gitlab-ci.yml` file to define your pipeline stages:\n\n\n```\n\nvariables:\n  TAG_LATEST: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME:latest\n  TAG_COMMIT: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHA\n\nstages:\n  - publish\n  - deploy\n```\n\n\nLet's break it down:\n\n\n`TAG_LATEST` is made up of three parts:\n\n\n- `$CI_REGISTRY_IMAGE` is the path to your project's container registry in\nGitLab\n\n\nFor example: `registry.gitlab.com/your-group/your-project`\n\n\n- `$CI_COMMIT_REF_NAME` is the name of your branch or tag\n\n\nFor example, if you're on main branch: `/main`, and if you're on a feature\nbranch: `/feature-login`\n\n\n- `:latest` is a fixed suffix\n\n\nSo if you're on the main branch, `TAG_LATEST` becomes:\n`registry.gitlab.com/your-group/your-project/main:latest`.\n\n\n`TAG_COMMIT` is almost identical, but instead of `:latest`, it uses:\n`$CI_COMMIT_SHA` which is the commit identifier, for example:\n`:abc123def456`.\n\n\nSo for that same commit on main branch, `TAG_COMMIT` becomes:`\nregistry.gitlab.com/your-group/your-project/main:abc123def456`.\n\n\nThe reason for having both is `TAG_LATEST` gives you an easy way to always\nget the newest version, and `TAG_COMMIT` gives you a specific version you\ncan return to if needed.\n\n\n#### 4. Publish to the container registry\n\n\nAdd the publish job to your pipeline:\n\n\n```\n\npublish:\n  stage: publish\n  image: docker:latest\n  services:\n    - docker:dind\n  script:\n    - docker build -t $TAG_LATEST -t $TAG_COMMIT .\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - docker push $TAG_LATEST\n    - docker push $TAG_COMMIT\n```\n\n\nThis job:\n\n\n- Uses Docker-in-Docker to build images\n\n- Creates two tagged versions of your image\n\n- Authenticates with the GitLab registry\n\n- Pushes both versions to the registry \n\n\nNow that our images are safely stored in the registry, we can focus on\ndeploying them to our target environments. Let's start with local testing to\nvalidate our setup before moving to production deployments.\n\n\n#### 5. Deploy to your environment\n\n\nBefore deploying to production, you can test locally. We just published our\nimage to the GitLab repository, which we’ll pull locally. If you’re unsure\nof the exact path, navigate to **Deploy > Container Registry**, and you\nshould see an icon to copy the path of your image at the end of the line for\nthe container image you want to test.\n\n\n```\n\ndocker login registry.gitlab.com \n\ndocker run -p 80:80 registry.gitlab.com/your-project-path/main:latest\n\n```\n\n\nBy doing so you should be able to access your application locally on your\nlocalhost address through your web browser.\n\n\nYou can now add a deployment job to your pipeline:\n\n\n```\n\ndeploy:\n  stage: deploy\n  image: alpine:latest\n  script:\n    - chmod 400 $GITLAB_KEY\n    - apk add openssh-client\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - ssh -i $GITLAB_KEY -o StrictHostKeyChecking=no $USER@$TARGET_SERVER \n      docker pull $TAG_COMMIT &&\n      docker rm -f myapp || true &&\n      docker run -d -p 80:80 --name myapp $TAG_COMMIT\n```\n\n\nThis job:\n\n\n- Sets up SSH access to your deployment target\n\n- Pulls the latest image\n\n- Removes any existing container\n\n- Deploys the new version\n\n\n#### 6. Track deployments\n\n\nEnable deployment tracking by adding environment configuration:\n\n\n```\n\ndeploy:\n  environment:\n    name: production\n    url: https://your-application-url.com \n```\n\n\nThis creates an environment object in GitLab's **Operate > Environments**\nsection, providing:\n\n\n- Deployment history\n\n- Current deployment status\n\n- Quick access to your application\n\n\nWhile a single environment pipeline is a good starting point, most teams\nneed to manage multiple environments for proper testing and staging. Let's\nexpand our pipeline to handle this more realistic scenario.\n\n\n#### 7. Set up multiple environments\n\n\nFor a more robust pipeline, configure staging and production deployments:\n\n\n```\n\nstages:\n  - publish\n  - staging\n  - release\n  - version\n  - production\n\nstaging:\n  stage: staging\n  rules:\n    - if: $CI_COMMIT_BRANCH == \"main\" && $CI_COMMIT_TAG == null\n  environment:\n    name: staging\n    url: https://staging.your-app.com\n  # deployment script here\n\nproduction:\n  stage: production\n  rules:\n    - if: $CI_COMMIT_TAG\n  environment:\n    name: production\n    url: https://your-app.com\n  # deployment script here\n```\n\n\nThis setup:\n\n\n- Deploys to staging from your main branch\n\n- Uses GitLab tags to trigger production deployments\n\n- Provides separate tracking for each environment\n\n\nHere and in our next step, we’re leveraging a very useful GitLab feature:\ntags. By manually creating a tag in the **Code > Tags** section, the\n`$CI_COMMIT_TAG` gets created, which allows us to trigger jobs accordingly.\n\n\n#### 8. Create automated release notes\n\n\nWe'll be using GitLab's release capabilities through our CI/CD pipeline.\nFirst, update your stages in `.gitlab-ci.yml`:\n\n\n```\n\nstages:\n\n\n- publish\n\n- staging\n\n- release # New stage for releases\n\n- version\n\n- production\n\n```\n\n\nNext, add the release job:\n\n\n```\n\nrelease_job:\n  stage: release\n  image: registry.gitlab.com/gitlab-org/release-cli:latest\n  rules:\n    - if: $CI_COMMIT_TAG                  # Only run when a tag is created\n  script:\n    - echo \"Creating release for $CI_COMMIT_TAG\"\n  release:                                # Release configuration\n    name: 'Release $CI_COMMIT_TAG'\n    description: 'Release created from $CI_COMMIT_TAG'\n    tag_name: '$CI_COMMIT_TAG'           # The tag to create\n    ref: '$CI_COMMIT_TAG'                # The tag to base release on\n```\n\n\nYou can enhance this by adding links to your container images:\n\n\n```\n\nrelease:\n  name: 'Release $CI_COMMIT_TAG'\n  description: 'Release created from $CI_COMMIT_TAG'\n  tag_name: '$CI_COMMIT_TAG'\n  ref: '$CI_COMMIT_TAG'\n  assets:\n    links:\n      - name: 'Container Image'\n        url: '$CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG'\n        link_type: 'image'\n```\n\n\nFor meaningful automated release notes:\n\n\n- Use conventional commits (feat:, fix:, etc.)\n\n- Include issue numbers (#123)\n\n- Separate subject from body with blank line\n\n\nIf you want custom release notes with deployment info:\n\n\n```\n\nrelease_job:\n  script:\n    - |\n      DEPLOY_TIME=$(date '+%Y-%m-%d %H:%M:%S')\n      CHANGES=$(git log $(git describe --tags --abbrev=0 @^)..@ --pretty=format:\"- %s\")\n      cat > release_notes.md \u003C\u003C EOF\n      ## Deployment Info\n      - Deployed on: $DEPLOY_TIME\n      - Environment: Production\n      - Version: $CI_COMMIT_TAG\n\n      ## Changes\n      $CHANGES\n\n      ## Artifacts\n      - Container Image: \\`$CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG\\`\n      EOF\n  release:\n    description: './release_notes.md'\n```\n\n\nOnce configured, releases will be created automatically when you create a\nGit tag. You can view them in GitLab under **Deploy > Releases**.\n\n\n#### 9. Put it all together\n\n\nThis is what our final YAML file looks like:\n\n\n```\n\nvariables:\n  TAG_LATEST: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME:latest\n  TAG_COMMIT: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHA\n  STAGING_TARGET: $STAGING_TARGET    # Set in CI/CD Variables\n  PRODUCTION_TARGET: $PRODUCTION_TARGET  # Set in CI/CD Variables\n\nstages:\n  - publish\n  - staging\n  - release\n  - version\n  - production\n\n# Build and publish to registry\n\npublish:\n  stage: publish\n  image: docker:latest\n  services:\n    - docker:dind\n  rules:\n    - if: $CI_COMMIT_BRANCH == \"main\" && $CI_COMMIT_TAG == null\n  script:\n    - docker build -t $TAG_LATEST -t $TAG_COMMIT .\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - docker push $TAG_LATEST\n    - docker push $TAG_COMMIT\n\n# Deploy to staging\n\nstaging:\n  stage: staging\n  image: alpine:latest\n  rules:\n    - if: $CI_COMMIT_BRANCH == \"main\" && $CI_COMMIT_TAG == null\n  script:\n    - chmod 400 $GITLAB_KEY\n    - apk add openssh-client\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - ssh -i $GITLAB_KEY -o StrictHostKeyChecking=no $USER@$STAGING_TARGET \"\n        docker pull $TAG_COMMIT &&\n        docker rm -f myapp || true &&\n        docker run -d -p 80:80 --name myapp $TAG_COMMIT\"\n  environment:\n    name: staging\n    url: http://$STAGING_TARGET\n\n# Create release\n\nrelease_job:\n  stage: release\n  image: registry.gitlab.com/gitlab-org/release-cli:latest\n  rules:\n    - if: $CI_COMMIT_TAG\n  script:\n    - |\n      DEPLOY_TIME=$(date '+%Y-%m-%d %H:%M:%S')\n      CHANGES=$(git log $(git describe --tags --abbrev=0 @^)..@ --pretty=format:\"- %s\")\n      cat > release_notes.md \u003C\u003C EOF\n      ## Deployment Info\n      - Deployed on: $DEPLOY_TIME\n      - Environment: Production\n      - Version: $CI_COMMIT_TAG\n\n      ## Changes\n      $CHANGES\n\n      ## Artifacts\n      - Container Image: \\`$CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG\\`\n      EOF\n  release:\n    name: 'Release $CI_COMMIT_TAG'\n    description: './release_notes.md'\n    tag_name: '$CI_COMMIT_TAG'\n    ref: '$CI_COMMIT_TAG'\n    assets:\n      links:\n        - name: 'Container Image'\n          url: '$CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG'\n          link_type: 'image'\n\n# Version the image with release tag\n\nversion_job:\n  stage: version\n  image: docker:latest\n  services:\n    - docker:dind\n  rules:\n    - if: $CI_COMMIT_TAG\n  script:\n    - docker pull $TAG_COMMIT\n    - docker tag $TAG_COMMIT $CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - docker push $CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG\n\n# Deploy to production\n\nproduction:\n  stage: production\n  image: alpine:latest\n  rules:\n    - if: $CI_COMMIT_TAG\n  script:\n    - chmod 400 $GITLAB_KEY\n    - apk add openssh-client\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - ssh -i $GITLAB_KEY -o StrictHostKeyChecking=no $USER@$PRODUCTION_TARGET \"\n        docker pull $CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG &&\n        docker rm -f myapp || true &&\n        docker run -d -p 80:80 --name myapp $CI_REGISTRY_IMAGE/main:$CI_COMMIT_TAG\"\n  environment:\n    name: production\n    url: http://$PRODUCTION_TARGET\n```\n\n\nThis complete pipeline:\n\n\n- Publishes images to the registry (main branch)\n\n- Deploys to staging (main branch)\n\n- Creates releases (on tags)\n\n- Versions images with release tags\n\n- Deploys to production (on tags)\n\n\nKey benefits:\n\n\n- Clean reproducible, local development and testing environment\n\n- Clear path to production environments with structure to build confidence\nin what is deployed\n\n- Pattern to recover from unexpected failures, etc.\n\n- Ready to scale/adopt more complex deployment strategies\n\n\n### Best practices\n\n\nThroughout implementation, maintain these principles:\n\n\n- Document everything, from variable usage to deployment procedures\n\n- Use GitLab's built-in features (environments, releases, registry)\n\n- Implement proper access controls and security measures\n\n- Plan for failure with robust rollback procedures\n\n- Keep your pipeline configurations DRY (Don't Repeat Yourself)\n\n\n## Scale your deployment strategy\n\n\nWhat next? Here are some aspects to consider as your continuous deployment\nstrategy matures.\n\n\n### Advanced security measures\n\n\nEnhance security through:\n\n\n- Protected environments with restricted access\n\n- Required approvals for production deployments\n\n- Integrated security scanning\n\n- Automated vulnerability assessments\n\n- Branch protection rules for deployment-related changes\n\n\n### Progressive delivery strategies\n\n\nImplement advanced deployment strategies:\n\n\n- Feature flags for controlled rollouts\n\n- Canary deployments for risk mitigation\n\n- Blue-green deployment strategies\n\n- A/B testing capabilities\n\n- Dynamic environment management\n\n\n### Monitoring and optimization\n\n\nEstablish robust monitoring practices:\n\n\n- Track deployment metrics\n\n- Set up performance monitoring\n\n- Configure deployment alerts\n\n- Establish deployment SLOs\n\n- Regular pipeline optimization\n\n\n## Why GitLab?\n\n\nGitLab's continuous deployment capabilities make it a standout choice for\nmodern deployment workflows. The platform excels in streamlining the path\nfrom code to production, offering built-in container registry, environment\nmanagement, and deployment tracking all within a single interface. GitLab's\nenvironment-specific variables, deployment approval gates, and rollback\ncapabilities provide the security and control needed for production\ndeployments, while features like review apps and feature flags enable\nprogressive delivery approaches. As part of GitLab's complete DevSecOps\nplatform, these CD capabilities seamlessly integrate with your entire\nsoftware lifecycle.\n\n\n## Get started today\n\n\nThe journey to continuous deployment is an evolution, not a revolution.\nStart with the fundamentals, build a solid foundation, and gradually\nincorporate advanced features as your team's needs grow. GitLab provides the\ntools and flexibility to support you at every stage of this journey, from\nyour first automated deployment to complex, multi-environment delivery\npipelines.\n\n\n> Sign up for a [free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/devsecops/) to get started\nwith continous deployment today.\n",[788,108,696,694,9],{"slug":1920,"featured":6,"template":700},"from-code-to-production-a-guide-to-continuous-deployment-with-gitlab","content:en-us:blog:from-code-to-production-a-guide-to-continuous-deployment-with-gitlab.yml","From Code To Production A Guide To Continuous Deployment With Gitlab","en-us/blog/from-code-to-production-a-guide-to-continuous-deployment-with-gitlab.yml","en-us/blog/from-code-to-production-a-guide-to-continuous-deployment-with-gitlab",{"_path":1926,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1927,"content":1933,"config":1938,"_id":1940,"_type":13,"title":1941,"_source":15,"_file":1942,"_stem":1943,"_extension":18},"/en-us/blog/get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1",{"title":1928,"description":1929,"ogTitle":1928,"ogDescription":1929,"noIndex":6,"ogImage":1930,"ogUrl":1931,"ogSiteName":685,"ogType":686,"canonicalUrls":1931,"schema":1932},"Get to know the security and governance updates in GitLab 17, 17.1","Dive deep into the new enhancements that can strengthen your organization's security posture, including how-to videos for SAST, DAST, API security, container registry, and more.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098858/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_282096522_securitycompliance.jpeg_1750098857843.jpg","https://about.gitlab.com/blog/get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Get to know the security and governance updates in GitLab 17, 17.1\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2024-07-17\",\n      }",{"title":1928,"description":1929,"authors":1934,"heroImage":1930,"date":1935,"body":1936,"category":830,"tags":1937},[1444],"2024-07-17","With every GitLab release we enhance and optimize security and governance\nsolutions to ensure customers have the tools they need to produce secure and\ncompliant software. Our values of\n[iteration](https://handbook.gitlab.com/handbook/values/#iteration) and\n[results for\ncustomers](https://handbook.gitlab.com/handbook/values/#results) drive our\nrelease cycles, and GitLab 17 is no exception. We have been releasing every\nmonth for the past 153 months straight!\n\n\nIn this article, you'll learn my favorite security and governance\nenhancements released in GitLab 17 and 17.1 and how they can benefit your\norganization’s security requirements. \n\n\n- [SAST analyzer streamlining](#sast-analyzer-streamlining)\n\n- [Android dependency scanning](#android-dependency-scanning)\n\n- [Custom roles and granular security permissions\nupdates](#custom-roles-and-granular-security-permissions-updates)\n\n- [Secret detection updates](#secret-detection-updates)\n\n- [Container registry updates](#container-registry-updates)\n\n- [API security scanning updates](#api-security-scanning-updates)\n\n\n## SAST analyzer streamlining\n\n\nGitLab provides static application security testing\n([SAST](https://docs.gitlab.com/ee/user/application_security/sast/)) to\nexamine your source code for known vulnerabilities, detecting\nvulnerabilities such as SQL injections and cross-site scripting. When SAST\nkicks off, the programming language used is auto-detected and the\nappropriate scanner is loaded.\n\n\nIn GitLab 17, SAST scans the same languages, but now with fewer analyzers,\n[offering a simpler and more customizable\nexperience](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#streamlined-sast-analyzer-coverage-for-more-languages).\nLanguage-specific analyzers have been replaced with GitLab-managed rules in\nthe Semgrep-based analyzer for the following languages:\n\n\n- C/C++\n\n- Swift (iOS)\n\n- Java/Kotlin (Android)\n\n- Node.js\n\n- PHP\n\n- Ruby\n\n\nHaving one analyzer for many different languages makes configurations and\nwriting rules easier than ever. See the [supported languages and frameworks\ndocumentation](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks)\nfor more information.\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/_80z6mZmzek?si=i9yPQttxuwVcb7Ye\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Android dependency scanning\n\n\nIn modern software development, many applications are built from multiple\ndependencies that are best at performing their intended function. For\nexample, rather than writing a YAML parser, a developer will use a library\nthat parses YAML. This allows developers to focus on the main goal of their\napplication, rather than spending time on utility functions.\n\n\nWhile the use of dependencies speeds up efficiency, they can be difficult to\nmanage and could introduce vulnerabilities to your application. For this,\nGitLab provides [dependency\nscanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/),\nwhich analyzes dependencies for known vulnerabilities. \n\n\nMany organizations are using dependencies even when creating native mobile\napplications. In GitLab 17, we introduced [Android dependency\nscanning](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#dependency-scanning-support-for-android)\nto bridge the gap. Android dependency scanning can be easily added as a\n[CI/CD catalog\ncomponent](https://gitlab.com/explore/catalog/components/android-dependency-scanning)\n– just include the following code in your `.gitlab-ci.yml`:\n\n\n```\n\ninclude:\n  - component: gitlab.com/components/android-dependency-scanning/component@1.0.0\n    inputs:\n      stage: test\n```\n\n\nThis job will also generate a CycloneDX software bill of materials\n([SBOM](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/))\nreport, which may be necessary for compliance. Make sure to scan your\nAndroid dependencies as soon as possible, as there are many CVEs out there.\n\n\nWatch this video to learn more:\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/_80z6mZmzek?si=DdB7j4NAenl-UcrJ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n> Learn more SBOMs and dependencies with [our ultimate guide to\nSBOMs](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/).\n\n\n## Custom roles and granular security permissions updates\n\n\nGitLab provides [custom\nroles](https://docs.gitlab.com/ee/user/custom_roles.html) to allow\norganizations to create user roles with the precise privileges and\npermissions to meet their needs. This enables organizations to [implement\nthe principle of least\nprivilege](https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab/)\nto adhere to various compliance standards.\n\n\n![custom roles\nscreenshot](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098874/Blog/Content%20Images/Blog/Content%20Images/1_aHR0cHM6_1750098873857.png)\n\n\nIn GitLab 17, managing custom roles has become easier than ever. You can now\n[edit a custom role and its permissions directly from the\nUI](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#edit-a-custom-role-and-its-permissions),\nwhereas, in the past, the role needed to be recreated. Also, for those using\nGitLab self-managed, [custom roles are now managed at the instance\nlevel](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#manage-custom-roles-at-self-managed-instance-level),\nallowing administrators to create the roles, and group owners to assign\nthem.\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/glvvCoc2hkc?si=dl_SwQ7tyVdzirH5\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nThere have also been [several UX\nimprovements](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#ux-improvements-to-custom-roles)\nadded to this feature along with the introduction of the following\npermissions:\n\n\n- assign security policy links\n\n- manage and assign compliance frameworks\n\n- manage webhooks\n\n- manage push rules\n\n- manage merge request settings (17.1)\n\n- manage integrations (17.1)\n\n- manage deploy tokens (17.1)\n\n- read CRM contacts (17.1)\n\n\nGitLab releases usually include new permissions to further enable the\nimplementation of the principle of least privilege. To learn more about the\navailable granular security permissions, [visit the available custom\npermission\ndocumentation](https://docs.gitlab.com/ee/user/custom_roles/abilities.html).\n\n\n## Secret detection updates\n\n\nDevelopers may accidentally commit secrets like keys or API tokens to Git\nrepositories from time to time. After a sensitive value is pushed to a\nremote repository, anyone with access to the repository can impersonate the\nauthorized user of the secret and cause mayhem. When this occurs the exposed\nsecrets must be revoked and replaced to address this risk, which can cause\nsystem downtime.\n\n\nGitLab provides [secret\ndetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/)\nto address this risk, and in GitLab 17 it’s gotten even better with the\nfollowing enhancements:\n\n\n- [Support for remote rulesets when overriding or disabling\nrules](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#secret-detection-now-supports-remote-rulesets-when-overriding-or-disabling-rules):\n- Allows you to override or disable rules via a remote configuration.\nTherefore, you can scale rule configurations across multiple projects using\nonly one [TOML](https://toml.io/en/) file.\n\n- [Advanced vulnerability\ntracking](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#introducing-advanced-vulnerability-tracking-for-secret-detection):\nDetects when the same secret has moved within a file due to refactoring or\nunrelated changes. This leads to reduced duplicate findings, simplifying\nvulnerability management.\n\n\nIn GitLab 17.1, [secret push\nprotection](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#secret-push-protection-available-in-beta)\nis now in Beta. Secret push protection checks the content of each commit\npushed to GitLab. If any secrets are detected, the push is blocked and\ndisplays information about the commit. Therefore, a developer does not need\nto do the extra work of removing and rotating secrets, since they are never\ncommitted upstream.\n\n\n![Push block eue to detected\nsecret](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098874/Blog/Content%20Images/Blog/Content%20Images/2_aHR0cHM6_1750098873858.png)\n\n\nWhen [push protection\noccurs](https://about.gitlab.com/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection/),\nyou can see it displays additional information on the commit, including:\n\n\n- the commit ID that contains the secret\n\n- the filename and line number that contains the secret\n\n- the type of secret\n\n\n**Note:**  [Enabling secret push\nprotection](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/#enable-secret-push-protection)\nis as easy as flipping a switch in GitLab Security Configuration.\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ZNtwXVj3tA8?si=4xJ1rWdThpVjvebv\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Container registry updates\n\n\nGitLab provides a [built-in container\nregistry](https://docs.gitlab.com/ee/user/packages/container_registry/),\nmaking it easy for developers to store and manage container images for each\nGitLab project without context switching. GitLab 17.1 includes several\nfeatures to enhance the security and efficiency of using the registry:\n\n- [Container images linked to\nsignatures](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#container-images-linked-to-signatures):\nContainer images in the registry can now be signed and associated with the\nsignature. This can reduce image tampering by allowing developers to quickly\nfind and validate the signatures that are associated with a container image\n\n- [Display the last published date for container\nimages](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#display-the-last-published-date-for-container-images):\nThe container registry UI has been updated to include accurate\n`last_published_at timestamps`, putting critical data at the top of view.\n\n- [Sort container registry tags by publish\ndate](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#sort-container-registry-tags-by-publish-date):\nAllows developers to quickly find and validate the most recently published\ncontainer image.\n\n\n![Signed container\ndetails](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098874/Blog/Content%20Images/Blog/Content%20Images/3_aHR0cHM6_1750098873860.png)\n\n\nAdditionally we’ve introduced [container scanning for the\nregistry](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#container-scanning-for-registry).\nThe container images being used in your application may themselves be based\non other container images that contain known vulnerabilities. Since\ndevelopers heavily make use of the built-in container registry, it is a\nno-brainer to introduce [container\nscanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/)\nfor the registry.\n\n\n[Container scanning for the\nregistry](https://docs.gitlab.com/ee/user/application_security/container_scanning/#container-scanning-for-registry)\ncan be easily enabled by flipping a switch in GitLab Security Configuration.\nOnce it’s enabled, whenever a container image is pushed to the container\nregistry in your project, GitLab checks its tag. If the tag is `latest`,\nthen GitLab creates a new pipeline that scans the image and even produces a\nCycloneDX SBOM.\n\n\n**Note:** At the moment, a vulnerability scan is only performed when a new\nadvisory is published. We are working to detract all vulnerabilities in the\nregistry itself in future iterations.\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Zuk7Axs-CRw?si=odlgT5HWv_KOnBtq\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## API security scanning updates\n\n\nWhile SAST does a great job of finding vulnerabilities in static source\ncode, there can still be vulnerabilities present in the running application\nthat cannot be detected in source code, such as broken authentication and\nsecurity misconfigurations. For these reasons, GitLab provides dynamic\napplication security testing\n([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)) and\n[Web API\nfuzzing](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/index.html)\nto help discover bugs and potential security issues that other QA processes\nmay miss. \n\n\nIn GitLab 17, we’ve introduced [several\nenhancements](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#api-security-testing-analyzer-updates)\nto our [dynamic scanners which target Web\nAPIs](https://docs.gitlab.com/ee/user/application_security/api_security_testing/index.html),\nincluding:\n\n- system environment variables are now passed from the CI runner to the\ncustom Python scripts used for certain advanced scenarios (like request\nsigning)\n\n- API Security containers now run as a non-root user, which improves\nflexibility and compliance\n\n- support for servers that only offer TLSv1.3 ciphers, which enables more\ncustomers to adopt API security testing.\n\n- scanner image upgraded to Alpine 3.19, which addresses security\nvulnerabilities\n\n\nIn GitLab 17.1, additional configuration variables were added to [API\nsecurity\nscanning](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#api-security-testing-analyzer-updates)\nand [API\nfuzzing](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#fuzz-testing-analyzer-updates)\nto allow:\n\n- creation of a comma-separated list of HTTP success status codes that\ndefine whether the job has passed\n\n- disabling of waiting for the target API to become available before\nscanning begins\n\n- specifying the expected status code for the API target availability check\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/CcyOoBgSPUU?si=hAMQfmUTlLRKhPSg\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Learn more about other enhancements\n\n\nGitLab 17 and 17.1 also introduced several other security and governance\nfeatures and enhancements, too many to cover in this blog. Some of these\nfeatures include:\n\n\n- [Updated filtering on the Vulnerability\nReport](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#updated-filtering-on-the-vulnerability-report):\nYou can now use the filtered search component to filter the Vulnerability\nReport by any combination of status, severity, tool, or activity.\n\n- [Toggle merge request approval policies to fail open or fail\nclosed](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#toggle-merge-request-approval-policies-to-fail-open-or-fail-closed):\nA new fail open option for merge request approval policies to offer\nflexibility to teams who want to ease the transition to policy enforcement\nas they roll out controls in their organization.\n\n- [Optional configuration for policy bot\ncomment](https://about.gitlab.com/releases/2024/05/16/gitlab-17-0-released/#optional-configuration-for-policy-bot-comment):\nThe security policy bot posts a comment on merge requests when they violate\na policy to help users understand when policies are enforced on their\nproject, when evaluation is completed, and if there are any violations\nblocking an MR, with guidance to resolve them.\n\n- [Merge request approval policies fail open/closed (policy\neditor)](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#merge-request-approval-policies-fail-openclosed-policy-editor):\nWithin the policy editor users can now toggle security policies to fail open\nor fail closed. This enhancement extends the YAML support to allow for\nsimpler configuration within the policy editor view.\n\n- [Project owners receive expiring access token\nnotifications](https://about.gitlab.com/releases/2024/06/20/gitlab-17-1-released/#project-owners-receive-expiring-access-token-notifications):\nBoth project owners and maintainers with direct membership now receive email\nnotifications when their project access tokens are close to expiring. This\nhelps keep more people informed about upcoming token expiration.\n\n\nThese are some of the newest security and compliance enhancements provided\nin GitLab 17 and 17.1 that can be applied to strengthen your organization's\nsecurity posture! To learn more about GitLab and the other ways we can\nstrengthen your organization's security throughout all parts of the software\ndevelopment lifecycle, check out the following links:\n\n\n- [GitLab Security and\nCompliance](https://about.gitlab.com/solutions/security-compliance/)\n\n- [GitLab Application Security\ndocumentation](https://docs.gitlab.com/ee/user/application_security/)\n\n- [GitLab security and governance overview\nvideo](https://youtu.be/Y4RC-SW8Ric)\n\n- [GitLab Complete DevSecOps\ndemo](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\n\n- [GitLab Complete DevSecOps\ntutorial](https://gitlab-da.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/) \n\n- [Ultimate guide to the principle of least\nprivilege](https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab/)\n",[830,694,9,495,696],{"slug":1939,"featured":90,"template":700},"get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1","content:en-us:blog:get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1.yml","Get To Know The Security And Governance Updates In Gitlab 17 17 1","en-us/blog/get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1.yml","en-us/blog/get-to-know-the-security-and-governance-updates-in-gitlab-17-17-1",{"_path":1945,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1946,"content":1952,"config":1956,"_id":1958,"_type":13,"title":1959,"_source":15,"_file":1960,"_stem":1961,"_extension":18},"/en-us/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab",{"title":1947,"description":1948,"ogTitle":1947,"ogDescription":1948,"noIndex":6,"ogImage":1949,"ogUrl":1950,"ogSiteName":685,"ogType":686,"canonicalUrls":1950,"schema":1951},"Getting started with GitLab: How to import your projects to GitLab","Learn how to import your projects from various sources, including Bitbucket, Gitea, GitHub, and GitLab Self-Managed.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097248/Blog/Hero%20Images/Blog/Hero%20Images/blog-getting-started-with-gitlab-banner-0497-option4-fy25_cFwd8DYFLekdnOLmbbChp_1750097247785.png","https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with GitLab: How to import your projects to GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2025-01-28\",\n      }",{"title":1947,"description":1948,"authors":1953,"heroImage":1949,"date":1544,"body":1954,"category":694,"tags":1955},[1604],"*Welcome to our \"Getting started with GitLab\" series, where we help\nnewcomers get familiar with the GitLab DevSecOps platform.*\n\n\nKnowing how to import your projects to GitLab is an essential skill to make\nthe most of the GitLab DevSecOps platform. You’ve [set up your\naccount](https://university.gitlab.com/pages/getting-started), invited\nusers, and\n[organized](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\nthem based on your use case or team structure. Now, you need to bring your\nexisting projects into GitLab and start collaborating. These projects can be\nlocal files on your computer or hosted on a different source code management\nplatform. Let's explore the options.\n\n\n## Importing local project files\n\n\nYou don't want to start from scratch every time you import a project. Follow\nthese steps to get into GitLab existing legacy projects or applications that\nexist without version control or use version control.\n\n\n### Git project\n\n\n1. If Git is [already\ninitiated](https://docs.gitlab.com/ee/topics/git/commands.html#git-init) in\nyour local project, create a new project in GitLab and obtain the SSH or\nHTTPS URL by clicking on the **Code** button in the top right corner of your\nproject page.\n\n\n![create a new project in GitLab with SSH/HTTPS\nURLs](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097254/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097252717.png)\n\n\n2. Switch to your terminal and ensure you are in your project folder:\n\n\n```bash  \n\ncd /project_folder  \n\n```\n\n\n3. Backup your existing [Git\norigin](https://git-scm.com/book/ms/v2/Git-Basics-Working-with-Remotes):\n\n\n```bash\n\n\ngit remote rename origin old-origin\n\n\n```\n\n\n4. Add the [GitLab\nremote](https://git-scm.com/book/ms/v2/Git-Basics-Working-with-Remotes) URL\nfor the new origin, when using SSH:\n\n\n```bash  \n\ngit remote add origin\n[git@gitlab.com](mailto:git@gitlab.com):gitlab-da/playground/abubakar/new-test-repo.git  \n\n```\n\n\nAnd for HTTPS: \n\n\n```bash  \n\ngit remote add origin\nhttps://gitlab.com/gitlab-da/playground/abubakar/new-test-repo.git  \n\n```\n\n\n5. Then push all existing\n[branches](https://docs.gitlab.com/ee/user/project/repository/branches/) and\n[tags](https://docs.gitlab.com/ee/user/project/repository/tags/) to GitLab:\n\n\n```bash  \n\ngit push --set-upstream origin --all  \n\ngit push --set-upstream origin --tags  \n\n```\n\n\nAll your file project files, branches, and tags will be pushed to GitLab and\nyou can start collaborating.\n\n\n### Non-Git project\n\n\nAlternatively, if you have not initiated Git in your project, you will need\nto initialize Git, commit existing files, and push to GitLab as follows:\n\n\n```bash  \n\ngit init --initial-branch=main  \n\ngit remote add origin\ngit@gitlab.com:gitlab-da/playground/abubakar/new-test-repo.git  \n\ngit add .  \n\ngit commit -m \"Initial commit\"  \n\ngit push --set-upstream origin main  \n\n```\n\n\n## Importing from online sources\n\n\nIf you have your project on GitLab.com or other platforms and you want to\nmove it to another GitLab instance (like a self-managed instance) or from\nanother platform to GitLab.com, GitLab provides the import project feature\nwhen you want to create a new project.\n\n\n![Create a new project\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097252718.png)\n\n\nImporting a project migrates the project files and some other components of\nthe project depending on the source. You can import from different sources\nlike Bitbucket, GitHub, Gitea, and a GitLab instance, among other sources.\nImport sources are enabled by default on GitLab.com, but they need to be\n[enabled for\nself-managed](https://docs.gitlab.com/ee/administration/settings/import_and_export_settings.html#configure-allowed-import-sources)\nby an administrator. We will look at a few of these sources in the following\nsections.\n\n\n![Import project from third-party\nsources](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097252719.png)\n\n\n## GitLab sources\n\n\nYou can export projects from GitLab.com and GitLab Self-Managed instances\nusing the Export project feature in a project’s settings. \n\n\n![Export project\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750097252720.png)\n\n\nTo access it:\n\n\n- Go to your project’s settings and click into the **General** area.\n\n- Scroll to and **Expand Advanced** section.\n\n- Select **Export project**.\n\n- A notification will be shown stating: “Project export started. A download\nlink will be sent by email and made available on this page.”\n\n- After the export is generated, you can follow the link contained in the\nemail or refresh the project settings page to reveal the “Download export”\noption.\n\n\n### Importing the project\n\n\n![Import an exported GitLab\nproject](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750097252722.png)\n\n\n- Click on the **New project** button in your target GitLab instance.  \n\n- Select **Import project** and click on **GitLab Export** in the list of\nimport sources.  \n\n- Specify a project name and select the export file, then click **Import\nproject**.  \n\n- An \"import in progress\" page will be shown and once complete, you will be\nredirected to the imported project.\n\n\nDepending on the size of your project, the import time may vary. It's\nimportant to note that not everything in a project might be exported and a\nfew things might change after import. Review the\n[documentation](https://docs.gitlab.com/ee/user/project/settings/import_export.html#export-a-project-and-its-data)\nto understand the limitations. If you want to migrate a whole group instead\nof individual projects, the [Direct Transfer\nmethod](https://docs.gitlab.com/ee/user/group/import/index.html) is\nrecommended; this creates a copy of an entire group.\n\n\n## Third-party providers\n\n\nGitLab supports importing from Bitbucket Cloud, Bitbucket Server, FogBugz,\nGitea, and GitHub. The import process is similar across all the supported\nthird parties — the main difference is in the method of authentication.\nLet's look at a few of them.\n\n\n### GitHub\n\n\n![Authenticate with GitHub\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097252723.png)\n\n\nThere are three methods to import GitHub projects in to GitLab:\n\n\n- [Using GitHub\nOAuth](https://docs.gitlab.com/ee/user/project/import/github.html#use-github-oauth)\n\n- [Using a GitHub personal access\ntoken](https://docs.gitlab.com/ee/user/project/import/github.html#use-a-github-personal-access-token)\n\n- [Using the\nAPI](https://docs.gitlab.com/ee/user/project/import/github.html#use-the-api)\n\n\nImporting using GitHub OAuth and personal access token are similar. The\ndifference lies in how your authorize GitLab to access your repositories.\nThe OAuth method is easier because you only need to click on the “Authorize\nwith GitHub” button and your are redirected to your GitHub account to\nauthorize the connection. Then the list of your projects is loaded for you\nto pick those you want to import.\n\n\n![Import repositories from GitHub\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097252725.png)\n\n\nAlternatively, you will need to generate a GitHub personal access token,\nselecting the `repo` and `read:org` scopes, and then provide it on the\n\"Import\" page.  For API imports, you can use the same personal access token\nwith our [Import REST API\nendpoints](https://docs.gitlab.com/ee/api/import.html#import-repository-from-github)\nin your script or application.\n\n\nIn this demo, GitLab Senior Developer Advocate Fernando Diaz explains how to\nimport a project from GitHub using the OAuth method:\n\n\n\u003C!-- blank line -->  \n\n\u003Cfigure class=\"video_container\"> \n  \u003Ciframe src=\"https://www.youtube.com/embed/0Id5oMl1Kqs?si=esF6wbz2j2JlhDVL\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>  \n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nYou can learn about prerequisites, known issues, importing from GitHub\nEnterprise, and other valuable information from the GitLab [import\ndocumentation](https://docs.gitlab.com/ee/user/project/import/github.html).\n\n\n### Bitbucket\n\n\nImporting projects from Bitbucket is similar to importing them from GitHub.\nWhile using OAuth is applicable to [Bitbucket\nCloud](https://docs.gitlab.com/ee/user/project/import/bitbucket.html), the\nSaaS version of Bitbucket, you'll need to provide a URL, username, and\npersonal access token for [Bitbucket\nServer](https://docs.gitlab.com/ee/user/project/import/bitbucket_server.html),\nthe enterprise self-hosted version. Clicking on the Bitbucket Cloud option\non the \"Import\" screen automatically takes you to Atlassian authentication\nfor Bitbucket.\n\n\n![Import project from\nBitBucket](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097252726.png)\n\n\nYou can also import Bitbucket projects using the [GitLab Import\nAPI](https://docs.gitlab.com/ee/api/import.html).\n\n\n### Gitea\n\n\n![Import project from\nGitea](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097252727.png)\n\n\nImporting projects from\n[Gitea](https://docs.gitlab.com/ee/user/project/import/gitea.html) requires\nthe creation of a [personal access\ntoken](https://docs.gitea.com/next/development/api-usage#authentication-via-the-api)\non the Gitea platform and providing it along with the Gitea server URL on\nthe GitLab import page. OAuth authentication is not supported. \n\n\n### Generic remote Git repository\n\n\n![Import project from remote Git\nrepository](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097253/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097252728.png)\n\n\nWhere your Git provider is not supported or import is not possible using the\nsupported methods, a repository can be imported using its accessible\n`https://` or `git://` URL.  If it's not publicly accessible, you will\nprovide the repository URL along with username and password (or access token\nwhere applicable due to multifactor authentication).\n\n\nThis method can also be used for maintaining a copy of a remote project and\nkeeping it in sync, i.e.,\n[mirroring](https://docs.gitlab.com/ee/user/project/repository/mirror/).\nMirroring allows you to maintain repositories across different platforms and\nkeep them synced. This can be to separate private and public access to\nproject while ensuring both ends have the same copy, which is useful when\nopen-sourcing  internal projects. It can also be used when working with\ncontractors and both parties use different platforms, and access to codebase\nis necessary on both ends. \n\n\n## Summary\n\n\nImporting and migrating between GitLab instances and from other sources is\nan important process that needs to be planned to ensure the expectations are\nclear on what gets imported and with which method. While most third-party\nmethods import project items, including files, issues, and merge requests,\nsome methods have known issues and limitations. The [GitLab import\nsection](https://docs.gitlab.com/ee/user/project/import/) of the\ndocumentation has detailed information on all the supported methods that can\nhelp you plan your migration.   \n\n\n> #### Want to take your learning to the next level? [Sign up for GitLab\nUniversity courses](https://university.gitlab.com/). Or you can get going\nright away with [a free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/devsecops/).\n\n\n## \"Getting started with GitLab\" series\n\n\n- [How to manage\nusers](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\n\n- [How to import your projects to\nGitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n\n- [Mastering project\nmanagement](https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management/)\n\n- [Automating Agile workflows with the gitlab-triage\ngem](https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem/)\n\n- [Working with CI/CD\nvariables](https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables/)\n",[694,9,495],{"slug":1957,"featured":6,"template":700},"getting-started-with-gitlab-how-to-import-your-projects-to-gitlab","content:en-us:blog:getting-started-with-gitlab-how-to-import-your-projects-to-gitlab.yml","Getting Started With Gitlab How To Import Your Projects To Gitlab","en-us/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab.yml","en-us/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab",{"_path":1963,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1964,"content":1970,"config":1975,"_id":1977,"_type":13,"title":1978,"_source":15,"_file":1979,"_stem":1980,"_extension":18},"/en-us/blog/getting-started-with-gitlab-how-to-manage-users",{"title":1965,"description":1966,"ogTitle":1965,"ogDescription":1966,"noIndex":6,"ogImage":1967,"ogUrl":1968,"ogSiteName":685,"ogType":686,"canonicalUrls":1968,"schema":1969},"Getting started with GitLab: How to manage users","Learn how to manage users using groups, roles, and permissions. Walk through the setup of secure collaboration with proper project access.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097273/Blog/Hero%20Images/Blog/Hero%20Images/blog-getting-started-with-gitlab-banner-0497-option4-fy25_cFwd8DYFLekdnOLmbbChp_1750097273817.png","https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with GitLab: How to manage users\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2025-01-14\",\n      }",{"title":1965,"description":1966,"authors":1971,"heroImage":1967,"date":1972,"body":1973,"category":694,"tags":1974},[1604],"2025-01-14","*Welcome to our \"Getting started with GitLab\" series, where we help newcomers get familiar with the GitLab DevSecOps platform.*\n\nEnsuring a safe, compliant, and collaborative environment starts with the most basic of tasks - managing users. In this tutorial, we show you how to establish project members, assign roles and permissions, and create groups and subgroups.\n\nNote: To follow along with this tutorial, you should have a GitLab account either through GitLab.com or your organization's self-managed instance. If you need help, visit our fundamentals area on [GitLab University](https://university.gitlab.com/).\n\nLet's get started.\n\nWhen you create GitLab users, they only have access to [their private projects, public projects, and projects set with internal visibility](https://docs.gitlab.com/ee/user/public_access.html). For the purposes of this tutorial, your project is super secret and only invited members should have access to it – at varying permissions settings. To ensure this, you can invite users as [members of the project](https://docs.gitlab.com/ee/user/project/members/).\n\n## Project members\n\n![Project members screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097278/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097278487.png)\n\nGitLab users can be invited to a project and [assigned a role](https://docs.gitlab.com/ee/user/permissions.html), which determines what they can do in the project. The owner of a project can delegate administrative tasks to other users as maintainers, who can do almost everything an owner does, aside from changes to a project such as deleting, archiving, or transferring a project.\n\n![Invite members screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097278/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097278487.png)\n\n[Maintainers](https://docs.gitlab.com/ee/user/permissions.html#roles) of the project can invite other members as developers who have access to all the features to create, build, and deploy software. Users who are not developers but need project management access can be invited to the project as [planners](https://about.gitlab.com/blog/introducing-gitlabs-new-planner-role-for-agile-planning-teams/), reporters, and guests with varying levels of permissions. These roles can also be used to determine who can make changes to certain branches with [protected branches](https://docs.gitlab.com/ee/user/project/repository/branches/protected.html).\n\nIf you are working with contractors or your use requires user permissions to expire, you can set an expiry date after which the user loses access to the project. Project members can also be identified as direct or indirect members, based on their [membership type](https://docs.gitlab.com/ee/user/project/members/#membership-types). Direct members are invited directly into the project, whereas indirect members are often inherited from a [GitLab group](https://docs.gitlab.com/ee/user/group/) a project belongs to.\n\nNow, let's look at Group memberships.\n\n## Group memberships\n\nGroups in GitLab can be a top level created at the root of a GitLab instance like the [gitlab.com/gitlab-org](http://gitlab.com/gitlab-org). which is a parent group used to organize other subgroups like [gitlab.com/gitlab-org/charts](http://gitlab.com/gitlab-org/charts). Groups are useful even if you only have one project.\n\nGroups can be used for different reasons:\n\n- organizing similar or related projects  \n- organizing users into groups for better team coordination\n\nWhen using groups to organize users, you can organize teams in groups and [invite a group to a project](https://docs.gitlab.com/ee/user/project/members/sharing_projects_groups.html) with a specific role for an entire team. You can have a `dev` group for the developers of the team, `pm` group for the project managers and `leads` for team leads. When inviting the groups, `dev` can be assigned the Developer role, `pm` the Planner role, and `leads` the Maintainer role. \n\n![Invite a group screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097279/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097278488.png)\n\nMembers of each group can be can be added or removed without needing to update project permissions. This is particularly useful when your team has grown to have several projects. However, it is important to [observe best practices](https://docs.gitlab.com/ee/user/project/members/sharing_projects_groups.html#setting-up-a-group-for-collaboration) for using groups for collaboration.\n\nAnother helpful aspect of having users organized in groups is that you can [mention](https://docs.gitlab.com/ee/user/discussions/#mentions) the entire group in issues, merge requests, or comments, which makes keeping an entire team informed easier.\n\n### Create subgroups\n\n[Subgroups](https://docs.gitlab.com/ee/user/group/subgroups/) can be used to further organize users in a group and you can keep adding subgroups up to 20 nested levels. Users in a subgroup inherit the the permissions they have in a parent group. If you want to grant a user in a subgroup a role higher than what they inherited, you will need to [invite them to the subgroup](https://docs.gitlab.com/ee/user/group/subgroups/#override-ancestor-group-membership) with the new higher role. Note: You can not give them a lower role in the subgroup.\n\n### Manage groups\n\nGroup Owners have several management options to determine how users function in a group. For instance, you can set how a user can request access to a group, enable/disable [group mentions](https://docs.gitlab.com/ee/user/group/manage.html#disable-group-mentions), [restrict access](https://docs.gitlab.com/ee/user/group/manage.html#turn-on-restricted-access), or [moderate users](https://docs.gitlab.com/ee/user/group/moderate_users.html), among other options. An exciting new feature, which is still under development at the time of this article's publication, is the [automatic removal of dormant users](https://docs.gitlab.com/ee/user/group/moderate_users.html#automatically-remove-dormant-members) after a minimum of 90 days and a maximum of five years. This will help keep groups clean and better manage the release of license seats.\n\n## Learn more\n\nManaging users on GitLab depends on your use case. If your organization is larger with more advanced workflows and user management, GitLab provides more advanced ways to [manage enterprise users](https://docs.gitlab.com/ee/user/enterprise_user/index.html). You can also explore more options on how to [manage your organization](https://docs.gitlab.com/ee/topics/set_up_organization.html) and with [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/), you get more granularity and compliance features.\n\n> #### Want to take your learning to the next level? [Sign up for GitLab University courses](https://university.gitlab.com/). Or you can get going right away with [a free trial of GitLab Ultimate](https://about.gitlab.com/free-trial/).\n\n## \"Getting started with GitLab\" series\nRead more articles in our \"Getting started with GitLab\" series:\n\n- [How to import your projects to GitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n- [Mastering project management](https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management/)\n- [Automating Agile workflows with the gitlab-triage gem](https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem/)\n- [Understanding CI/CD](https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd/)\n- [Working with CI/CD variables](https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables/)\n",[495,696,9,996,694],{"slug":1976,"featured":90,"template":700},"getting-started-with-gitlab-how-to-manage-users","content:en-us:blog:getting-started-with-gitlab-how-to-manage-users.yml","Getting Started With Gitlab How To Manage Users","en-us/blog/getting-started-with-gitlab-how-to-manage-users.yml","en-us/blog/getting-started-with-gitlab-how-to-manage-users",{"_path":1982,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":1983,"content":1989,"config":1994,"_id":1996,"_type":13,"title":1997,"_source":15,"_file":1998,"_stem":1999,"_extension":18},"/en-us/blog/getting-started-with-gitlab-mastering-project-management",{"title":1984,"description":1985,"ogTitle":1984,"ogDescription":1985,"noIndex":6,"ogImage":1986,"ogUrl":1987,"ogSiteName":685,"ogType":686,"canonicalUrls":1987,"schema":1988},"Getting started with GitLab: Mastering project management","Discover the key components of project management and how to put them to use for better organization and tracking.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097294/Blog/Hero%20Images/Blog/Hero%20Images/blog-getting-started-with-gitlab-banner-0497-option4-fy25_cFwd8DYFLekdnOLmbbChp_1750097293924.png","https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with GitLab: Mastering project management\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2025-02-11\",\n      }",{"title":1984,"description":1985,"authors":1990,"heroImage":1986,"date":1991,"body":1992,"category":694,"tags":1993},[868],"2025-02-11","*Welcome to our \"Getting started with GitLab\" series, where we help newcomers get familiar with the GitLab DevSecOps platform.*\n\nGitLab is much more than just a place to store your code. It is an AI-powered DevSecOps platform with tools to help you plan, organize, track, and successfully deliver your projects. This post will guide you through GitLab's key project management features and show you how to leverage them effectively.\n\n## Why GitLab for project management?\n\nImagine having your code repository, issue tracker, and communication platform all seamlessly integrated in one place. That's the power of GitLab. By centralizing everything, you can streamline your workflow, enhance collaboration, and keep your projects moving forward. No more jumping between different tools and losing track of information. GitLab brings it all together, making it easier to manage your projects from start to finish.\n\n## Key components of GitLab project management\n\nLet's break down the essential elements:\n\n* [Epics](https://docs.gitlab.com/ee/user/group/epics/): Think of epics as the big picture. They represent major features, overarching goals, or long-term initiatives within your project. Need to revamp your website? That's an epic! Epics help you organize your work into larger, manageable chunks.  \n* [Issues](https://docs.gitlab.com/ee/user/project/issues/): Issues are the individual tasks or work items that contribute to your project goals. Each issue represents a specific action, like \"design the homepage\" or \"write the 'about us' page.\" Issues are the building blocks of your project, and they provide a clear way to track individual tasks.  \n* [Labels](https://docs.gitlab.com/ee/user/project/labels.html): Labels are like tags that help you categorize and filter your work. You can use labels to indicate priority (e.g., high, medium, low), status (e.g., to do, in progress, done), or assign issues to specific teams or individuals. Labels provide a flexible way to organize and prioritize your work.  \n* Boards: GitLab's issue boards are your visual workspace. They provide a Kanban-style view of your project, allowing you to see the status of all your issues at a glance. Drag and drop issues across different lists (e.g., \"To Do,\" \"Doing,\" \"Done\") to visualize your workflow and track progress. In GitLab, you can create boards for [issues](https://docs.gitlab.com/ee/user/project/issue_board.html) and [Epics](https://docs.gitlab.com/ee/user/group/epics/epic_boards.html).  \n* [Milestones](https://docs.gitlab.com/ee/user/project/milestones/): Milestones mark significant checkpoints or target dates within your project. They help you track progress towards specific goals and deadlines. For example, you might have milestones for completing a major feature, releasing a beta version, or launching the final product.  \n* [Tasks](https://docs.gitlab.com/ee/user/tasks.html): For those extra granular steps, break down your issues into smaller tasks. This helps with delegation, clarifies individual responsibilities, and ensures nothing gets overlooked. Tasks provide a way to create checklists within issues, making it easier to track progress on complex tasks.\n\n## Deep dive into the features\n\n### 1. Epics: The big picture\n\n* Creating epics: Navigate to your group's \"Epics\" menu under “Plan”. Click **New epic** and give it a descriptive title and a clear description outlining the goal. You can also specify the start and end date of the epic – this is useful when using [Roadmaps](https://docs.gitlab.com/ee/user/group/roadmap/).\n\n![Epic creation page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097300817.png)\n\n* [Roadmaps](https://docs.gitlab.com/ee/user/group/roadmap/): Add your epics to a roadmap to visualize your project timeline and long-term goals. Roadmaps provide a bird's-eye view of your project plan, making it easy to see the big picture and track progress towards major milestones.\n\n![Roadmap view](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097300818.png)\n\n### 2. Issues: Getting things done\n\n* Creating issues: In your project, go to the \"Issues\" menu under “Plan” and click **New issue**. Provide a concise and descriptive title like \"Design Homepage Wireframes,\" assign it to a team member, set a due date, and add a detailed description outlining the task's requirements.  \n* GitLab Duo: You can leverage the power of [GitLab Duo to create detailed issue descriptions](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#populate-an-issue-with-issue-description-generation) with just a little hint of what you want to achieve.  \n* Weighting: Estimate the effort required for each issue by assigning weights. This helps with planning and prioritization. For example, a simple task might have a weight of **1**, while a more complex task might have a weight of **5**.\n\n![Issue with weight of 4 assigned](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097300819.png)\n\n### 3. Labels: Organizing your work\n\n* Creating labels: Go to your project's \"Issues\" tab and click Labels. Create custom labels with clear names to categorize your issues. For example, create labels like **Priority: High**, **Status: In Progress**, or **Team: Design**. Apply these labels to your issues to keep them organized and easily filterable.\n\n![Labels screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097300820.png)\n\n### 4. Boards: Visualizing your workflow\n\n* Kanban boards: GitLab's boards provide a Kanban-style view of your project. Create lists like \"To Do,\" \"Doing,\" and \"Done\" to represent the stages of your workflow. Drag and drop issues across these lists to visualize their progress.\n* Customizing boards: Tailor your boards to match your specific workflow. Add more columns, filter issues by labels or assignees, and set up swim lanes to categorize issues by epics or other criteria.\n\n![Visualize workflow with issue boards](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097300820.png)\n\n### 5. Tasks: Breaking down the work\n\n* Creating tasks: Within an issue, use the checklist markdown syntax to create a task list. Each item in the list represents a smaller step within the larger issue. For example, in the issue \"Design Homepage Wireframes,\" you might have tasks like \"Sketch initial concepts,\" \"Create digital wireframes,\" and \"Get feedback from stakeholders.\" To create a Task, click on the **Add** button in the \"Child Items\" section of an issue’s page. Then, enter the title of the task, and click **Create Task**.\n\n![Issue with create task button](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097300822.png)\n\n### 6. Milestones: Tracking progress\n* Setting milestones: Define milestones to mark significant points in your project, like completing a specific feature or reaching a key deadline. Give your milestones clear titles and due dates.\n* Associating with issues: Link issues and epics to milestones to track progress towards those goals. This helps you see how individual tasks contribute to the overall project plan.\nCreating a milestone: Under the \"Plan\" dropdown menu, click on **Milestones > New milestone**. Specify the milestone title, description, and start and due dates.\n\n![New milestone screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097300823.png)\n\n\u003Cbr>\u003C/br>\n\n![New page wtih milestone on it](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097300823.png)\n\n### 7. [Iterations](https://docs.gitlab.com/ee/user/group/iterations/): Working in sprints\n\n* Defining iterations: If you're using an Agile workflow, define iterations (sprints) with specific start and end dates. This helps you break down your work into smaller, more manageable time boxes.  \n* Assigning issues: Assign issues to iterations to plan your work in shorter cycles and focus on delivering incremental value.\n\n### 8. [Time tracking](https://docs.gitlab.com/ee/user/project/time_tracking.html): Measuring effort\n\n* Logging time: Within an issue, use the \"/spend\" quick action followed by the time spent (e.g., \"/spend 2h 30m\") to log your work. This helps you track the actual time spent on each task.  \n* Analyzing data: Generate time tracking reports to gain insights into project progress, team efficiency, and identify potential bottlenecks.\n\n![Time tracking report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097301/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750097300824.png)\n\n### 9. Dependencies: Managing workflow\n\n* [Linking issues](https://docs.gitlab.com/ee/user/project/issues/related_issues.html): Create dependencies between issues to ensure tasks are completed in the correct order. For example, if issue A must be completed before issue B can begin, you can create a dependency between them. This helps you visualize the workflow and avoid potential roadblocks.\n\n### 10. Templates: Streamlining issue creation\n\n* [Creating templates](https://docs.gitlab.com/ee/user/project/description_templates.html): Create issue templates to standardize the information captured for common tasks, saving you time and ensuring consistency. For example, you could create a template for bug reports that includes fields for steps to reproduce expected behavior and actual behavior.\n\n### Collaboration is key\n\nGitLab fosters collaboration through the following:\n\n* [Comments](https://docs.gitlab.com/ee/user/discussions/): Discuss issues and epics directly within GitLab. Use comments to provide updates, ask questions, and share feedback.  \n* [Mentions](https://docs.gitlab.com/ee/user/discussions/#mentions): Use **@** to mention specific team members and notify them of updates or request their input.  \n* Discussions: Engage in threaded discussions within issues and epics to brainstorm ideas, solve problems together, and keep everyone informed.\n\n### Ready to get started?\n\nNow that you've explored the power of GitLab's project management features, it's time to put them into practice! Create a sample project, experiment with different features, and discover how GitLab can transform your workflow. You can also learn more about how GitLab can help you facilitate [Kanban](https://docs.gitlab.com/ee/tutorials/kanban/) and [Scrum](https://docs.gitlab.com/ee/tutorials/scrum_events/) in the GitLab documentation.\n\n> #### Want to take your learning to the next level? [Sign up for GitLab University courses](https://university.gitlab.com/). Or you can get going right away with a [free trial of GitLab Ultimate](https://about.gitlab.com/free-trial/).\n\n## \"Getting started with GitLab\" series\nRead more articles in our \"Getting started with GitLab\" series:\n\n- [How to manage users](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\n- [How to import your projects to GitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n- [Automating Agile workflows with the gitlab-triage gem](https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem/)\n- [Understanding CI/CD](https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd/)\n- [Working with CI/CD variables](https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables/)",[9,694,495,996],{"slug":1995,"featured":6,"template":700},"getting-started-with-gitlab-mastering-project-management","content:en-us:blog:getting-started-with-gitlab-mastering-project-management.yml","Getting Started With Gitlab Mastering Project Management","en-us/blog/getting-started-with-gitlab-mastering-project-management.yml","en-us/blog/getting-started-with-gitlab-mastering-project-management",{"_path":2001,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2002,"content":2007,"config":2012,"_id":2014,"_type":13,"title":2015,"_source":15,"_file":2016,"_stem":2017,"_extension":18},"/en-us/blog/getting-started-with-gitlab-understanding-ci-cd",{"title":2003,"description":2004,"ogTitle":2003,"ogDescription":2004,"noIndex":6,"ogImage":988,"ogUrl":2005,"ogSiteName":685,"ogType":686,"canonicalUrls":2005,"schema":2006},"Getting started with GitLab: Understanding CI/CD","Learn the basics of continuous integration/continuous delivery in this beginner's guide, including what CI/CD components are and how to create them.","https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with GitLab: Understanding CI/CD\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2025-04-25\",\n      }",{"title":2003,"description":2004,"authors":2008,"heroImage":988,"date":2009,"body":2010,"category":694,"tags":2011},[868],"2025-04-25","*Welcome to our \"Getting started with GitLab\" series, where we help\nnewcomers get familiar with the GitLab DevSecOps platform.*\n\n\nImagine a workflow where every code change is automatically built, tested,\nand deployed to your users. That's the power of [Continuous\nIntegration/Continuous Delivery\n(CI/CD)](https://about.gitlab.com/topics/ci-cd/)! CI/CD helps you catch bugs\nearly, ensures code quality, and delivers software faster and more\nfrequently.\n\n\n### What is CI/CD?\n\n\n* **Continuous Integration** is a development practice where developers\nintegrate code changes into a shared repository frequently, preferably\nseveral times a day. Each integration is then verified by an automated build\nand test process, allowing teams to detect problems early.  \n\n* **Continuous Delivery** extends CI by automating the release pipeline,\nensuring that your code is *always* in a deployable state. You can deploy\nyour application to various environments (e.g., staging, production) with a\nsingle click or automatically.  \n\n* **Continuous Deployment** takes it a step further by automatically\ndeploying *every successful build* to production. This requires a high\ndegree of confidence in your automated tests and deployment process.\n\n\n### Why GitLab CI/CD?\n\n\nGitLab CI/CD is a powerful, integrated system that comes built-in with\nGitLab. It offers a seamless experience for automating your entire software\ndevelopment lifecycle. With GitLab CI/CD, you can:\n\n\n* **Automate everything:** Build, test, and deploy your applications with\nease.  \n\n* **Catch bugs early:** Detect and fix errors before they reach\nproduction.  \n\n* **Get faster feedback:** Receive immediate feedback on your code\nchanges.  \n\n* **Improve collaboration:** Work together more effectively with automated\nworkflows.  \n\n* **Accelerate delivery:** Release software faster and more frequently.  \n\n* **Reduce risk:** Minimize deployment errors and rollbacks.\n\n\n### The elements of GitLab CI/CD\n\n\n* `.gitlab-ci.yml`**:** This [YAML\nfile](https://docs.gitlab.com/ee/ci/yaml/), located in your project's root\ndirectory, defines your CI/CD pipeline, including stages, jobs, and\nrunners.  \n\n* [**GitLab Runner**](https://docs.gitlab.com/runner/)**:** This agent\nexecutes your CI/CD jobs on your infrastructure (e.g. physical machines,\nvirtual machines, Docker containers, or Kubernetes clusters).  \n\n* [**Stages**](https://docs.gitlab.com/ee/ci/yaml/#stages)**:** Stages\ndefine the order of execution for your jobs (e.g. build, test, and\ndeploy).  \n\n* [**Jobs**](https://docs.gitlab.com/ee/ci/yaml/#job-keywords)**:** Jobs are\nindividual units of work within a stage (e.g. compile code, run tests, and\ndeploy to staging).\n\n\n### Setting up GitLab CI\n\n\nGetting started with GitLab CI is simple. Here's a basic example of a\n`.gitlab-ci.yml` file:\n\n\n```yaml\n\n\nstages:\n  - build\n  - test\n  - deploy\n\nbuild_job:\n  stage: build\n  script:\n    - echo \"Building the application...\"\n\ntest_job:\n  stage: test\n  script:\n    - echo \"Running tests...\"\n\ndeploy_job:\n  stage: deploy\n  script:\n    - echo \"Deploying to production...\"\n  environment:\n    name: production\n\n```\n\n\nThis configuration defines three stages: \"build,\" \"test,\" and \"deploy.\" Each\nstage contains a job that executes a simple script.\n\n\n### CI/CD configuration examples\n\n\nLet's explore some more realistic examples.\n\n\n**Building and deploying a Node.js application**\n\n\nThe pipeline definition below outlines using npm to build and test a Node.js\napplication and [dpl](https://docs.gitlab.com/ci/examples/deployment/) to\ndeploy the application to Heroku. The deploy stage of the pipeline makes use\nof [GitLab CI/CD variables](https://docs.gitlab.com/ci/variables/), which\nallow developers to store sensitive information (e.g. credentials) and\nsecurely use them in CI/CD processes. In this example, an API key to deploy\nto Heroku is stored under the variable key name `$HEROKU_API_KEY` used by\nthe dpl tool.\n\n\n```yaml\n\n\nstages:\n  - build\n  - test\n  - deploy\n\nbuild:\n  stage: build\n  image: node:latest\n  script:\n    - npm install\n    - npm run build\n\ntest:\n  stage: test\n  image: node:latest\n  script:\n    - npm run test\n\ndeploy:\n  stage: deploy\n  image: ruby:latest\n  script:\n    - gem install dpl\n    - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_API_KEY\n\n```\n\n\n**Deploying to different environments (staging and production)**\n\n\nGitLab also offers the idea of\n[Environments](https://docs.gitlab.com/ci/environments/) with CI/CD. This\nfeature allows users to track deployments from CI/CD to infrastructure\ntargets. In the example below, the pipeline adds stages with an environment\nproperty for a staging and production environment. While the deploy_staging\nstage will always run its script, the deploy_production stage requires\nmanual approval to prevent accidental deployment to production.  \n\n\n```yaml\n\n\nstages:\n  - build\n  - test\n  - deploy_staging\n  - deploy_production\n\nbuild:\n  # ...\n\ntest:\n  # ...\n\ndeploy_staging:\n  stage: deploy_staging\n  script:\n    - echo \"Deploying to staging...\"\n  environment:\n    name: staging\n\ndeploy_production:\n  stage: deploy_production\n  script:\n    - echo \"Deploying to production...\"\n  environment:\n    name: production\n  when: manual  # Requires manual approval\n\n```\n\n\n### GitLab Auto DevOps\n\n\n[GitLab Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/)\nsimplifies CI/CD by providing a pre-defined configuration that automatically\nbuilds, tests, and deploys your applications. It leverages best practices\nand industry standards to streamline your workflow.\n\n\nTo enable Auto DevOps:\n\n\n1. Go to your project's **Settings > CI/CD > General pipelines**.  \n\n2. Enable the **Auto DevOps** option.\n\n\nAuto DevOps automatically detects your project's language and framework and\nconfigures the necessary build, test, and deployment stages. You don’t even\nneed to create a `.gitlab-ci.yml` file.\n\n\n### CI/CD Catalog\n\n\nThe [CI/CD\nCatalog](https://about.gitlab.com/blog/faq-gitlab-ci-cd-catalog/)\nis a list of projects with published [CI/CD\ncomponents](https://docs.gitlab.com/ee/ci/components/) you can use to extend\nyour CI/CD workflow. Anyone can create a component project and add it to the\nCI/CD Catalog or contribute to an existing project to improve the available\ncomponents. You can find published components in the [CI/CD\nCatalog](https://gitlab.com/explore/catalog) on GitLab.com.\n\n\n> [Tutorial: How to set up your first GitLab CI/CD\ncomponent](https://about.gitlab.com/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component/)\n\n\n### CI templates\n\n\nYou can also create your own [CI\ntemplates](https://docs.gitlab.com/ee/ci/examples/) to standardize and reuse\nCI/CD configurations across multiple projects. This promotes consistency and\nreduces duplication.\n\n\nTo create a CI template:\n\n\n1. Create a `.gitlab-ci.yml` file in a dedicated project or repository.  \n\n2. Define your CI/CD configuration in the template.  \n\n3. In your project's `.gitlab-ci.yml` file, use the `include` keyword to\ninclude the template.\n\n\n## Take your development to the next level\n\n\nGitLab CI/CD is a powerful tool that can transform your development\nworkflow. By understanding the concepts of CI/CD, configuring your\npipelines, and leveraging features like Auto DevOps, the CI/CD Catalog, and\nCI templates, you can automate your entire software development lifecycle\nand deliver high-quality software faster and more efficiently.\n\n\n> Want to take your learning to the next level? Sign up for [GitLab\nUniversity courses](https://university.gitlab.com/). Or you can get going\nright away with a [free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).\n\n\n## \"Getting Started with GitLab\" series\n\n\nCheck out more articles in our \"Getting Started with GitLab\" series:\n\n\n- [How to manage\nusers](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\n\n- [How to import your projects to\nGitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n\n- [Mastering project\nmanagement](https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management/)\n\n- [Automating Agile workflows with the gitlab-triage\ngem](https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem/)\n\n- [Working with CI/CD\nvariables](https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables/)\n",[108,787,788,495,694,9],{"slug":2013,"featured":90,"template":700},"getting-started-with-gitlab-understanding-ci-cd","content:en-us:blog:getting-started-with-gitlab-understanding-ci-cd.yml","Getting Started With Gitlab Understanding Ci Cd","en-us/blog/getting-started-with-gitlab-understanding-ci-cd.yml","en-us/blog/getting-started-with-gitlab-understanding-ci-cd",{"_path":2019,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2020,"content":2025,"config":2031,"_id":2033,"_type":13,"title":2034,"_source":15,"_file":2035,"_stem":2036,"_extension":18},"/en-us/blog/getting-started-with-gitlab-working-with-ci-cd-variables",{"title":2021,"description":2022,"ogTitle":2021,"ogDescription":2022,"noIndex":6,"ogImage":988,"ogUrl":2023,"ogSiteName":685,"ogType":686,"canonicalUrls":2023,"schema":2024},"Getting started with GitLab: Working with CI/CD variables","Learn what CI/CD variables are, why they are important in DevSecOps, and best practices for utilizing them.","https://about.gitlab.com/blog/getting-started-with-gitlab-working-with-ci-cd-variables","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with GitLab: Working with CI/CD variables\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab Team\"}],\n        \"datePublished\": \"2025-05-27\",\n      }",{"title":2021,"description":2022,"authors":2026,"heroImage":988,"date":2028,"body":2029,"category":694,"tags":2030},[2027],"GitLab Team","2025-05-27","*Welcome to our \"Getting started with GitLab\" series, where we help\nnewcomers get familiar with the GitLab DevSecOps platform.*\n\n\nIn an earlier article, we explored [GitLab\nCI/CD](https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd/).\nNow, let's dive deeper into the world of **CI/CD variables** and unlock\ntheir full potential.\n\n\n### What are CI/CD variables?\n\n\nCI/CD variables are dynamic key-value pairs that you can define at different\nlevels within your GitLab environment (e.g., project, group, or instance).\nThese variables act as placeholders for values that you can use in your\n`.gitlab-ci.yml` file to customize your pipelines, securely store sensitive\ninformation, and make your CI/CD configuration more maintainable.\n\n\n### Why are CI/CD variables important?\n\n\nCI/CD variables offer numerous benefits:\n\n\n* **Flexibility** - Easily adapt your pipelines to different environments,\nconfigurations, or deployment targets without modifying your core CI/CD\nscript.  \n\n* **Security** - Securely store sensitive information like API keys,\npasswords, and tokens, preventing them from being exposed directly in your\ncode.  \n\n* **Maintainability** - Keep your CI/CD configuration clean and organized by\ncentralizing values in variables, making updates and modifications easier.  \n\n* **Reusability** - Define variables once and reuse them across multiple\nprojects, promoting consistency and reducing duplication.\n\n\n### Scopes of CI/CD variables: Project, group, and instance\n\n\nGitLab allows you to define CI/CD variables with different scopes,\ncontrolling their visibility and accessibility:\n\n\n* **Project-level variables** - These variables are specific to a single\nproject and are ideal for storing project-specific settings, such as:\n  * Deployment URLs: Define different URLs for staging and production environments.  \n  * Database credentials: Store database connection details for testing or deployment.  \n  * Feature flags: Enable or disable features during different stages of your pipeline.  \n  * Example: You have a project called \"MyWebApp\" and want to store the production deployment URL. You create a project-level variable named `DPROD_DEPLOY_URL` with the value `https://mywebapp.com`.  \n* **Group-level variables** - These variables are shared across all projects\nwithin a GitLab group. They are useful for settings that are common to\nmultiple projects, such as:\n\n  * API keys for shared services: Store API keys for services like AWS, Google Cloud, or Docker Hub that are used by multiple projects within the group.  \n  * Global configuration settings: Define common configuration parameters that apply to all projects in the group.  \n  * Example: You have a group called \"Web Apps\" and want to store an API key for Docker Hub. You create a group-level variable named `DOCKER_HUB_API_KEY` with the corresponding API key value.  \n* **Instance-level variables** - These variables are available to all\nprojects on a GitLab instance. They are typically used for global settings\nthat apply across an entire organization such as:\n\n  * Default runner registration token: Provide a default token for registering new [runners](https://docs.gitlab.com/runner/).  \n  * License information: Store license keys for GitLab features or third-party tools.  \n  * Global environment settings: Define environment variables that should be available to all projects.  \n  * Example: You want to set a default Docker image for all projects on your GitLab instance. You create an instance-level variable named `DEFAULT_DOCKER_IMAGE` with the value `ubuntu:latest`.\n\n### Defining CI/CD variables\n\n\nTo define a CI/CD variable:\n\n\n1. Click on the **Settings > CI/CD** buttons for  your project, group, or\ninstance.  \n\n2. Go to the **Variables** section.  \n\n3. Click **Add variable**.  \n\n4. Enter the **key** (e.g., `API_KEY`) and **value**.  \n\n5. Optionally, check the **Protect variable** box for sensitive information.\nThis ensures that the variable is only available to pipelines running on\nprotected branches or tags.  \n\n6. Optionally, check the **Mask variable** box to hide the variable's value\nfrom job logs, preventing accidental exposure.  \n\n7. Click **Save variable**.\n\n\n### Using CI/CD variables\n\n\nTo use a CI/CD variable in your `.gitlab-ci.yml` file, simply prefix the\nvariable name with `$`:\n\n\n```yaml\n\ndeploy_job:\n  script:\n    - echo \"Deploying to production...\"\n    - curl -H \"Authorization: Bearer $API_KEY\" https://api.example.com/deploy\n```\n\n\n### Predefined CI/CD variables\n\n\nGitLab provides a set of [predefined CI/CD\nvariables](https://docs.gitlab.com/ci/variables/predefined_variables/) that\nyou can use in your pipelines. These variables provide information about the\ncurrent pipeline, job, project, and more.\n\n\nSome commonly used predefined variables include:\n\n\n* `$CI_COMMIT_SHA`: The commit SHA of the current pipeline.  \n\n* `$CI_PROJECT_DIR`: The directory where the project is cloned.  \n\n* `$CI_PIPELINE_ID`: The ID of the current pipeline.  \n\n* `$CI_ENVIRONMENT_NAME`: The name of the environment being deployed to (if\napplicable).\n\n\n### Best practices\n\n\n* Securely manage sensitive variables: Use protected and masked variables\nfor API keys, passwords, and other sensitive information.  \n\n* Avoid hardcoding values: Use variables to store configuration values,\nmaking your pipelines more flexible and maintainable.  \n\n* Organize your variables: Use descriptive names and group related variables\ntogether for better organization.  \n\n* Use the appropriate scope: Choose the correct scope (project, group, or\ninstance) for your variables based on their intended use and visibility.\n\n\n### Unlock the power of variables\n\n\nCI/CD variables are a powerful tool for customizing and securing your GitLab\npipelines. By mastering variables and understanding their different scopes,\nyou can create more flexible, maintainable, and efficient workflows.\n\n\nWe hope you found it helpful and are now well-equipped to leverage the power\nof GitLab for your development projects.\n\n\n> Get started with CI/CD variables today with a [free trial of\nGitLab Ultimate with Duo Enterprise](https://about.gitlab.com/free-trial/).\n\n\n## \"Getting Started with GitLab\" series\n\nRead more articles in our \"Getting Started with GitLab\" series:\n\n\n- [How to manage\nusers](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-manage-users/)\n\n-  [How to import your projects to\nGitLab](https://about.gitlab.com/blog/getting-started-with-gitlab-how-to-import-your-projects-to-gitlab/)  \n\n- [Mastering project\nmanagement](https://about.gitlab.com/blog/getting-started-with-gitlab-mastering-project-management/)\n\n- [Automating Agile workflows with the gitlab-triage\ngem](https://about.gitlab.com/blog/automating-agile-workflows-with-the-gitlab-triage-gem/)\n\n- [Understanding\nCI/CD](https://about.gitlab.com/blog/getting-started-with-gitlab-understanding-ci-cd/)\n",[694,9,787,788,108,696],{"slug":2032,"featured":90,"template":700},"getting-started-with-gitlab-working-with-ci-cd-variables","content:en-us:blog:getting-started-with-gitlab-working-with-ci-cd-variables.yml","Getting Started With Gitlab Working With Ci Cd Variables","en-us/blog/getting-started-with-gitlab-working-with-ci-cd-variables.yml","en-us/blog/getting-started-with-gitlab-working-with-ci-cd-variables",{"_path":2038,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2039,"content":2045,"config":2051,"_id":2053,"_type":13,"title":2054,"_source":15,"_file":2055,"_stem":2056,"_extension":18},"/en-us/blog/getting-started-with-value-streams-dashboard",{"title":2040,"description":2041,"ogTitle":2040,"ogDescription":2041,"noIndex":6,"ogImage":2042,"ogUrl":2043,"ogSiteName":685,"ogType":686,"canonicalUrls":2043,"schema":2044},"Getting started with the new GitLab Value Streams Dashboard","Benchmark your value stream lifecycle, DORA, and vulnerabilities metrics to gain valuable insights and uncover patterns for continuous improvements.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749671793/Blog/Hero%20Images/16_0-cover-image.png","https://about.gitlab.com/blog/getting-started-with-value-streams-dashboard","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Getting started with the new GitLab Value Streams Dashboard\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Haim Snir\"}],\n        \"datePublished\": \"2023-06-12\",\n      }",{"title":2040,"description":2041,"authors":2046,"heroImage":2042,"date":2048,"body":2049,"category":785,"tags":2050},[2047],"Haim Snir","2023-06-12","\n\n\u003Ci>This is part two of our multipart series introducing you to the capabilities within GitLab Value Stream Management and the Value Streams Dashboard. In part one, [learn about the Total Time Chart](https://about.gitlab.com/blog/value-stream-total-time-chart/) and how to simplify top-down optimization flow with Value Stream Management.\u003C/i>\n\nGetting started with GitLab [Value Streams Dashboard](https://docs.gitlab.com/ee/user/analytics/value_streams_dashboard.html), a customizable dashboard that enables decision-makers to identify trends, patterns, and opportunities for digital transformation improvements, is easy. If you're already using GitLab Value Stream Management, simply navigate to your project's or group's Analytics tab, and within [Value stream analytics](https://docs.gitlab.com/ee/user/group/value_stream_analytics/#view-value-stream-analytics), click on the \"Value Streams Dashboard - DORA\" link. This will open a new page with the Value Streams Dashboard.\n\n![image of DORA Metrics console](https://about.gitlab.com/images/blogimages/vsdCover.png){: .shadow}\nDORA metrics comparison panel\n{: .note.text-center}\n\nGitLab Value Stream Management allows customers to visualize their end-to-end DevSecOps workstreams, manage their software development processes, and gain insight into how digital transformation and technological investments are delivering value and driving business results. GitLab Value Stream Management is able to do this because GitLab provides an entire DevOps platform as a single application and, therefore, holds all the data needed to provide end-to-end visibility throughout the entire software development lifecycle. So now your decisions rely on actual data rather than blind estimation or gut feelings. Additionally, because GitLab is the place where work happens, GitLab Value Stream Management insights are also actionable, allowing your users to move from \"understanding\" to \"fixing\" at any time, from within their workflow and without losing context.\n\nThe centralized UI in Value Streams Dashboard acts as the single source of truth (SSOT), where all stakeholders can access and view the same set of metrics that are relevant to the organization. The SSOT views ensure consistency, eliminate discrepancies, and provide a reliable and unified source of data for decision-making and analysis.\n\nThe first iteration of the GitLab Value Streams Dashboard was focused on enabling teams to continuously improve software delivery workflows by benchmarking [value stream lifecycle metrics, DORA metrics, and vulnerabilities metrics](https://docs.gitlab.com/ee/user/analytics/value_streams_dashboard.html#dashboard-metrics-and-drill-down-reports). One of the key features is a new DevSecOps metrics comparison panel that displays the metrics for a group or project in the month-to-date, last month, the month before, and the past 180 days.\n\nThis comparison enables managers to track team improvements in the context of the other DevSecOps metrics to find patterns or trends over time. The data is presented in a clear and concise manner, ensuring that you can quickly grasp the significance of the metrics.\n\n![The Value Streams Dashboard helps you get a high-level custom view over multiple DevOps metrics and understand whether they are improving month-over-month](https://about.gitlab.com/images/blogimages/2023-05-18_vsd_1.gif){: .shadow}\nValue Streams Dashboard metrics comparison panel\n{: .note.text-center}\n\nAdditionally, from each metric you can drill down to a detailed report to investigate the underlying data, understand what is affecting the team performance, and identify actionable insights.\n\nWe understand that every organization has its own set of subgroups and projects, each with specific processes and terminology. That's why we designed our dashboard to be flexible and adaptable. Users have the power to [customize](https://docs.gitlab.com/ee/user/analytics/value_streams_dashboard.html#customize-the-dashboard-panels) their dashboard by including panels from different subgroups or projects. \n\nTracking and comparing these metrics over a period of time helps teams catch downward trends early, drill down into individual projects/metrics, take remedial actions to maintain their software delivery performance, and track progress of their innovation investments. Value Streams Dashboard's intuitive interface reduces the learning curve and eliminates the need for extensive training. Everyone can now immediately leverage the platform's unified data store power, maximizing their productivity and saving precious time and resources.\n\n## Value Streams Dashboard roadmap\nWe are just getting started with delivering new capabilities in our Value Streams Dashboard. The roadmap includes planned features and functionality that will continue to improve decision-making and operational efficiencies.\n\nSome of the capabilities we plan to focus on next include:\n\n- adding an [executive-level summary](https://gitlab.com/groups/gitlab-org/-/epics/9558) of key metrics related to software performance and flow of value across the organization\n- adding a [\"DORA Performers score\"](https://gitlab.com/groups/gitlab-org/-/epics/10416) panel with the DORA metrics health from all the organization's groups and projects\n- adding [filter by label to the comparison panel](https://gitlab.com/gitlab-org/gitlab/-/issues/388890) - we recognize that every team does not follow the same flow so we are adding them to slice and dice the dashboard views with GitLab labels as filters\n\nTo help us improve the Value Stream Management Dashboard, please share feedback about your experience in this [survey](https://gitlab.fra1.qualtrics.com/jfe/form/SV_50guMGNU2HhLeT4).\n\n## Learn more\n* Find out what's next on the [Value Stream Management direction page](https://about.gitlab.com/direction/plan/value_stream_management/#whats-next-and-why).\n\n* Learn how to use the new dashboard using the [Value Streams Dashboard documentation](https://docs.gitlab.com/ee/user/analytics/value_streams_dashboard.html).\n\n* Watch this short video on Value Streams Dashboards:\n\n\u003Ciframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/EA9Sbks27g4\" frameborder=\"0\" allow=\"accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen>\u003C/iframe>\n\nCheck out part three of this multipart series: \"[GitLab's 3 steps to optimizing software value streams](https://about.gitlab.com/blog/three-steps-to-optimize-software-value-streams/)\".\n\n\u003Ci>Disclaimer: This blog contains information related to upcoming products, features, and functionality. It is important to note that the information in this blog post is for informational purposes only. Please do not rely on this information for purchasing or planning purposes. As with all projects, the items mentioned in this blog and linked pages are subject to change or delay. The development, release, and timing of any products, features, or functionality remain at the sole discretion of GitLab.\u003C/i>\n",[9,696,744,956,996],{"slug":2052,"featured":6,"template":700},"getting-started-with-value-streams-dashboard","content:en-us:blog:getting-started-with-value-streams-dashboard.yml","Getting Started With Value Streams Dashboard","en-us/blog/getting-started-with-value-streams-dashboard.yml","en-us/blog/getting-started-with-value-streams-dashboard",{"_path":2058,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2059,"content":2065,"config":2070,"_id":2072,"_type":13,"title":2073,"_source":15,"_file":2074,"_stem":2075,"_extension":18},"/en-us/blog/github-to-gitlab-migration-made-easy",{"title":2060,"description":2061,"ogTitle":2060,"ogDescription":2061,"noIndex":6,"ogImage":2062,"ogUrl":2063,"ogSiteName":685,"ogType":686,"canonicalUrls":2063,"schema":2064},"GitHub to GitLab migration the easy way","Learn how easy it is to migrate from GitHub to GitLab using GitLab's project import functionality.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749668776/Blog/Hero%20Images/julia-craice-faCwTallTC0-unsplash.jpg","https://about.gitlab.com/blog/github-to-gitlab-migration-made-easy","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitHub to GitLab migration the easy way\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2023-07-11\",\n      }",{"title":2060,"description":2061,"authors":2066,"heroImage":2062,"date":2067,"body":2068,"category":1119,"tags":2069},[1444],"2023-07-11","If you are using different CI/CD tools and are considering migrating over to\nGitLab, you may be wondering about\n\nthe difficulty of the migration process. Migration is usually a concern for\n[DevSecOps](https://about.gitlab.com/topics/devsecops/) teams when\nconsidering a new solution. This is due to the fact that migrating may\ninvolve heavy lifting. However, migrating to the GitLab AI-powered DevSecOps\nPlatform can be extremely simple and I will show you how step by step. \n\n\nIn this blog post, we will go over how to migrate from GitHub to GitLab\nusing our [project import](https://docs.gitlab.com/ee/user/project/import/)\nfunctionality. Manually migrating GitHub Actions to GitLab pipelines will be\ncovered as well. I have also created a video going over the migration\nprocess for those who prefer that format:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/0Id5oMl1Kqs\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## What data can be migrated from GitHub to GitLab?\n\nGitLab's built-in importer allows for GitHub projects to be automatically\nmigrated into GitLab. The built-in importer\n\nis accessed directly from GitLab's project creation UI. From the UI, you can\nselect what data you wish to migrate to GitLab.\n\n\nThe data that can be migrated includes the following:\n\n* Repository description\n\n* Git repository data\n\n* Branch protection rules\n\n* Collaborators (members)\n\n* Issues\n\n* Pull requests\n\n* Wiki pages\n\n* Milestones\n\n* Labels\n\n* Release notes content\n\n* Release notes attachments\n\n* Comment attachments\n\n* Issue description attachments\n\n* Pull request description attachments\n\n* Pull request review comments\n\n* Regular issue and pull request comments\n\n* Git Large File Storage (LFS) objects\n\n* Pull request reviews\n\n* Pull request assigned reviewers\n\n* Pull request “merged by” information\n\n* Pull request comments replies in discussions\n\n* Pull request review comments suggestions\n\n* Issue events and pull requests events\n\n\nGitHub and GitLab have different naming conventions and concepts, so a\nmapping must be performed during the migration. For example, when\ncollaborators/members are migrated, roles from GitHub are mapped to the\nappropriate GitLab roles as follows:\n\n\n| GitHub role | GitLab role |\n\n| ----------- | ----------- |\n\n| Read        | Guest       |\n\n| Triage      | Reporter    |\n\n| Write       | Developer   |\n\n| Maintain    | Maintainer  |\n\n| Admin       | Owner       |\n\n\n## Prerequisites\n\nNow that you have an understanding of what can be imported, let's review the\nprerequisites for performing the migration.\n\n\nWith the GitLab importer, you can either import your projects from\n**GitHub.com** or **GitHub Enterprise** to either **GitLab.com** or\n**Self-managed GitLab** as long as you meet the following requirements:\n\n* You must be a Maintainer on the GitLab destination group you are importing\nto from GitHub\n\n* Each GitHub author and assignee in the repository must have a\npublic-facing email address on GitHub that matches their GitLab email\naddress\n\n* GitHub accounts must have a public-facing email address that is populated\n\n* [GitHub import\nsource](https://docs.gitlab.com/ee/administration/settings/visibility_and_access_controls.html#configure-allowed-import-sources)\nmust be enabled (Self-managed GitLab only)\n\n\nWhen migrating a user, GitLab uses the public-facing email address in GitHub\nto verify the user with the same email on GitLab. Because email ownership is\nunique, you'll know you have set a valid user with valid permissions.\n\n\n## Performing the import\n\nNow let's go over how to perform the migration. I will be migrating my\nproject, the [Reddit sentiment\nanalyzer](https://github.com/fishtoadsoft/reddit-sentiment-analyzer), from\nGitHub to GitLab. The Reddit sentiment analyzer contains a pull request\n(called a merge request in GitLab), issues, and comments. \n\n\n**Note:** While you may not have permissions to my project, the step-by-step\nprocess applies to any project you own. I am using my project so you can see\nhow I migrate GitHub Actions in the next section. Now, let's get started!\n\n\n1) Create a new project in GitLab using the [Project Creation\nInterface](https://gitlab.com/projects/new).\n\n\n2) Select the **Import Project** box. This allows you to migrate data from\nexternal sources.\n\n\n![Import project\nbox](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/import_project.png)\n\n\n3) Under **Import project from**, press the **GitHub** button. This will\ntake you to the **Authenticate with GitHub** page.\n\n\n4) Press the **Authenticate with GitHub** button. You can also use a\n[personal access\ntoken](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)\nfrom GitHub with the **repo scope** if you prefer. This will take you to the\nGitHub authorization app.\n\n\n5) From here, you can grant access to [GitHub\norganization(s)](https://docs.github.com/en/organizations/collaborating-with-groups-in-organizations/about-organizations)\nwhere the projects you wish to migrate are located.\n\n\n![GitHub authorization\napp](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/github_authorize_app.png)\n\n\n6) Press the **Grant** button for the organization where the project you\nwish to migrate is stored.\n\n\n7) Press the **Authorize gitlabhq** button to grant GitLab access to the\norganization(s) selected. You will then be taken to the import selection\npage.\n\n\n8) From here, you can select the items you wish to import. \n\n\n![Import\nselection](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/import_selection.png)\n\n\n**Note:** The more items you choose to migrate, the longer the import will\ntake.\n\n\n9) Then you must set the GitLab location you want to migrate the GitHub\nproject to.\n\n\n![Set the GitLab location to migrate\nto](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/import_to.png)\n\n\n10) Press the **Import** button and the import will begin. You can see the\nprogress in the UI. Once the import is complete the status will be changed\nto \"complete.\"\n\n\n[Import progress\nstatus](/images/blogimages/2023-july-github-to-gitlab-migration/import_progress.png)\n\n\nNow you should have the imported project in your workspace. Mine is called\n[https://gitlab.com/awkwardferny/reddit-sentiment-analyzer](https://gitlab.com/awkwardferny/reddit-sentiment-analyzer).\nWhen examining the imported project, you can see the following:\n\n\n**Repository has been migrated**\n\n\n![Repository has been\nmigrated](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/migrated_data.png)\n\n\n**Issue has been migrated**\n\n\n![Issue has been\nmigrated](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/migrated_issue.png)\n\n\n**Merge request has been migrated**\n\n\n![Merge request has been\nmigrated](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/migrated_merge_request.png)\n\n\n## Migrating GitHub Actions over to GitLab CI/CD\n\nNow that you have migrated the project over from GitHub, notice that none of\nthe GitHub Actions are running. Don't worry, they are very easy to migrate\nmanually. So let's start the migration process for Actions.\n\n\n1) Examine the GitHub Actions within the **.github/workflows** folder. In\nthe [project you just\nimported](https://gitlab.com/awkwardferny/reddit-sentiment-analyzer/-/tree/master/.github/workflows),\nyou should see three different Action files:\n\n\n#### lint.yml\n\nThis file contains the Action, which performs linting on the source code\nusing flake8. It uses the python:3.10 Docker image and installs the\napplication requirements before performing the lint.\n\n\n```yaml\n\nname: \"Lint\"\n\n\non:\n  push:\n    branches: [ master ]\n  pull_request:\n    branches: [ master ]\n\njobs:\n  lint:\n    runs-on: ubuntu-latest\n    steps:\n    - uses: actions/checkout@v3\n    - name: Set up Python 3.10\n      uses: actions/setup-python@v4\n      with:\n        python-version: \"3.10\"\n    - name: Install dependencies\n      run: |\n        python -m pip install --upgrade pip\n        pip install flake8 pytest\n        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n    - name: Lint with flake8\n      run: |\n        # stop the build if there are Python syntax errors or undefined names\n        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics\n        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide\n        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics\n```\n\n\n#### smoke.yml\n\nThis file contains the action which performs a smoke test by just running\nthe CLI help menu. It uses the python:3.10 Docker image and installs the\napplication requirements before performing the smoke test.\n\n\n```yaml\n\nname: \"Smoke Tests\"\n\n\non:\n  push:\n    branches: [ master ]\n  pull_request:\n    branches: [ master ]\n\njobs:\n  smoke-tests:\n    runs-on: ubuntu-latest\n    steps:\n    - uses: actions/checkout@v3\n    - name: Set up Python 3.10\n      uses: actions/setup-python@v4\n      with:\n        python-version: \"3.10\"\n    - name: Install dependencies\n      run: |\n        python -m pip install --upgrade pip\n        pip install setuptools\n        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n    - name: Install Sentiment Analysis Application\n      run: |\n        python setup.py install\n    - name: Run smoke tests\n      run: |\n        reddit-sentiment --help\n```\n\n\n#### unit.yml\n\nThis file contains the Action, which performs unit tests using pytest. It\nuses the python:3.10 Docker image and installs the application requirements\nrunning the unit tests.\n\n\n```yaml\n\nname: \"Unit Tests\"\n\n\non:\n  push:\n    branches: [ master ]\n  pull_request:\n    branches: [ master ]\n\njobs:\n  unit-tests:\n    runs-on: ubuntu-latest\n    steps:\n    - uses: actions/checkout@v3\n    - name: Set up Python 3.10\n      uses: actions/setup-python@v4\n      with:\n        python-version: \"3.10\"\n    - name: Install dependencies\n      run: |\n        python -m pip install --upgrade pip\n        pip install pytest\n        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n    - name: Test with pytest\n      run: |\n        python -m pip install --upgrade pip\n        if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi\n        pytest tests/\n```\n\n\nNow let's go ahead and migrate these Actions over to GitLab.\n\n\n2) Go to the recently imported project on GitLab and open up the\n[WebIDE](https://docs.gitlab.com/ee/user/project/web_ide/).\n\n\n3) Create a file at the root called\n[**.gitlab-ci.yml**](https://docs.gitlab.com/ee/ci/yaml/gitlab_ci_yaml.html).\n\nThis file defines the GitLab pipeline.\n\n\n4) Add the following configuration, which will add the GitHub Actions as\nJobs in the GitLab pipeline. Notice the comments I added describing each\nsection.\n\n\n```yaml\n\n# This creates the stages in which the jobs will run. By default all\n\n# jobs will run in parallel in the stage. Once the jobs are completed\n\n# successfully then you move on to the next stage. The way jobs run\n\n# is completely configurable.\n\nstages:\n  - test\n\n# With the include statement, you can quickly add jobs which have\n\n# been pre-defined in external YAMLs. The SAST job I included below\n\n# is provided and maintained by GitLab and adds Static Application\n\n# Security Testing (SAST) to your pipeline.\n\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n\n# This is the unit test job which does exactly what is defined in\n\n# the GitHub Action in unit.yml. You can see it uses the python:3.10\n\n# Docker image, installs the application dependencies, and then runs\n\n# the unit tests with pytest. It was added with a simple copy and\n\n# paste and minor syntax changes.\n\nunit:\n  image: python:3.10\n  stage: test\n  before_script:\n    - python -m pip install --upgrade pip\n    - pip install pytest\n    - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n  script:\n    - pytest tests/\n\n# This is the lint job which does exactly what is defined in the\n\n# GitHub Action in lint.yml. You can see it uses the python:3.10\n\n# Docker image, installs the application dependencies, and then\n\n# performs the linting with flake8. It was added with a simple copy\n\n# and paste and minor syntax changes.\n\nlint:\n  image: python:3.10\n  stage: test\n  before_script:\n    - python -m pip install --upgrade pip\n    - pip install flake8\n    - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n  script:\n    - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics\n    - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics\n\n# This is the smoke test job which does exactly what is defined in\n\n# the GitHub Action in smoke.yml. You can see it uses the python:3.10\n\n# Docker image, installs the application dependencies, and then runs\n\n# the smoke tests with the Reddit sentiment analysis CLI. It was\n\n# added with a simple copy and paste and minor syntax changes.\n\nsmoke:\n  image: python:3.10\n  stage: test\n  before_script:\n    - python -m pip install --upgrade pip\n    - pip install setuptools\n    - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n    - python setup.py install\n  script:\n    - reddit-sentiment --help\n```\n\n\nYou can see that scripts being executed in GitLab match those scripts within\nthe GitHub Actions. The only thing that has really changed is the syntax\nsetting up the jobs and stages. To learn more on how to create and configure\npipelines, check out the [GitLab CI/CD\ndocumentation](https://docs.gitlab.com/ee/ci/).\n\n\n5) Let's check in the code. From the WebIDE click on the Source Control Tab\nin the side panel of the WebIDE. It is the [third icon from the\ntop](https://code.visualstudio.com/docs/sourcecontrol/overview#_commit).\nThen press the **Commit to 'main'** button, select **Continue**, and voila,\nyou should now have a running pipeline.\n\n\n6) Examine the pipeline and make sure the jobs are running properly. Go back\nto your project and click on the\n[pipeline](https://docs.gitlab.com/ee/ci/pipelines/) icon. You can see the\nthe four jobs we created have run.\n\n\n![Four jobs have\nrun](https://about.gitlab.com/images/blogimages/2023-july-github-to-gitlab-migration/gitlab_jobs.png)\n\n\n7) Click on the **Unit** job and you can see that the unit tests were run\nsuccessfully.\n\n\n```bash\n\n$ pytest tests/\n\n============================= test session starts\n==============================\n\nplatform linux -- Python 3.10.11, pytest-7.3.1, pluggy-1.0.0\n\nrootdir: /builds/awkwardferny/reddit-sentiment-analyzer\n\ncollected 2 items\n\ntests/test_scraper.py ..                                                \n[100%]\n\n============================== 2 passed in 0.09s\n===============================\n\nCleaning up project directory and file based variables\n\n00:00\n\nJob succeeded\n\n```\n\n\nAnd that's how simple it is to migrate a project over from GitHub to GitLab!\n\n\n## What other platforms can GitLab import from?\n\nThe GitLab importer allows one-click migration from several other platforms.\nThese platforms include:\n\n* [Bitbucket\nCloud](https://docs.gitlab.com/ee/user/project/import/bitbucket.html)\n\n* [Bitbucket Server\n(Stash)](https://docs.gitlab.com/ee/user/project/import/bitbucket_server.html)\n\n* [FogBugz](https://docs.gitlab.com/ee/user/project/import/fogbugz.html)\n\n* [Gitea](https://docs.gitlab.com/ee/user/project/import/gitea.html)\n\n* [Repository by\nURL](https://docs.gitlab.com/ee/user/project/import/repo_by_url.html)\n\n* [Uploading a manifest file\n(AOSP)](https://docs.gitlab.com/ee/user/project/import/manifest.html)\n\n* [Jira (issues\nonly)](https://docs.gitlab.com/ee/user/project/import/jira.html)\n\n\nWe also have documentation covering how to migrate from these platforms:\n\n*\n[SVN](https://docs.gitlab.com/ee/user/project/import/#import-from-subversion)\n\n* [ClearCase](https://docs.gitlab.com/ee/user/project/import/clearcase.html)\n\n* [CVS](https://docs.gitlab.com/ee/user/project/import/cvs.html)\n\n* [Perforce](https://docs.gitlab.com/ee/user/project/import/perforce.html)\n\n* [TFVC](https://docs.gitlab.com/ee/user/project/import/tfvc.html)\n\n\n---\n\n\nThanks for reading! Now you know how easy it is to migrate from GitHub over\nto GitLab. For more information on GitLab\n\nand migrating from GitHub, follow the links below:\n\n\n* [GitHub-to-GitLab project migration\ndocumentation](https://docs.gitlab.com/ee/user/project/import/github.html)\n\n* [Available project\nimporters](https://docs.gitlab.com/ee/user/project/import/#available-project-importers)\n\n* [GitHub-to-GitLab migration video](https://youtu.be/0Id5oMl1Kqs)\n\n\nAlso, read how GitLab has been named a leader in the DevOps platforms space\nby\n[Gartner](https://about.gitlab.com/blog/gitlab-leader-gartner-magic-quadrant-devops-platforms/)\nand the integrated software delivery platforms space by\n[Forrester](https://about.gitlab.com/blog/gitlab-leader-forrester-wave-integrated-software-delivery-platforms/).\n\n\n_Cover image by [Julia\nCraice](https://unsplash.com/@jcraice?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/s/photos/migration?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)_\n",[108,9,851,495],{"slug":2071,"featured":6,"template":700},"github-to-gitlab-migration-made-easy","content:en-us:blog:github-to-gitlab-migration-made-easy.yml","Github To Gitlab Migration Made Easy","en-us/blog/github-to-gitlab-migration-made-easy.yml","en-us/blog/github-to-gitlab-migration-made-easy",{"_path":2077,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2078,"content":2084,"config":2089,"_id":2091,"_type":13,"title":2092,"_source":15,"_file":2093,"_stem":2094,"_extension":18},"/en-us/blog/gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes",{"title":2079,"description":2080,"ogTitle":2079,"ogDescription":2080,"noIndex":6,"ogImage":2081,"ogUrl":2082,"ogSiteName":685,"ogType":686,"canonicalUrls":2082,"schema":2083},"GitLab Duo + Amazon Q: Transform ideas into code in minutes","The new GitLab Duo with Amazon Q integration analyzes your issue descriptions and automatically generates complete working code solutions, accelerating development workflows.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097127/Blog/Hero%20Images/Blog/Hero%20Images/Screenshot%202024-11-27%20at%204.55.28%E2%80%AFPM_4VVz6DgGBOvbGY8BUmd068_1750097126673.png","https://about.gitlab.com/blog/gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab Duo + Amazon Q: Transform ideas into code in minutes\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2025-04-28\",\n      }",{"title":2079,"description":2080,"authors":2085,"heroImage":2081,"date":2086,"body":2087,"category":718,"tags":2088},[715],"2025-04-28","Have you ever spent days or even weeks converting a complex issue into working code? We've all been there. You start with a solid idea and a clear set of requirements, but the path from that initial concept to deployable code can be frustratingly long. Your productivity gets bogged down in implementation details, and projects that should move quickly end up dragging on.\n\nThis is where the power of [agentic AI](https://about.gitlab.com/topics/agentic-ai/) capabilities comes in. [GitLab Duo with Amazon Q](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/), which combines the comprehensive AI-powered DevSecOps platform with the deepest set of cloud computing capabilities, is designed to dramatically accelerate your application development process, all within your familiar GitLab workflow. By streamlining your path from idea to deployment, this powerful integration can propose implementation solutions based on your issue descriptions alone – transforming what used to take days into something that happens in minutes.\n\n## How it works: From issue to working code\n\nLet's walk through how this agentic AI feature works in practice. Imagine you're a developer tasked with creating a mortgage calculator application. Here's how GitLab Duo with Amazon Q helps you get it done:\n\n1. **Create an issue with detailed requirements:** Start by creating a standard [GitLab issue](https://docs.gitlab.com/user/project/issues/). In the description, you'll provide a comprehensive list of requirements that your service needs to meet. This becomes the blueprint for your solution.\n\n2. **Invoke Amazon Q with a quick action:** Once your issue is created, simply add a comment with a quick action, “/q dev”, to invoke Amazon Q. This is where the magic begins. \n\n3. **Let AI generate your implementation:** GitLab Duo with Amazon Q analyzes the issue description you've provided and the context of your source code, then autonomously generates code that meets all your stated requirements. It doesn't stop there – it actually commits those changes in a merge request, ready for your review.\n\n![GitLab Duo  with Amazon Q activity pop-up screenshot](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097156/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097156018.png)\n\n4. **Review the generated application**: Navigate into the merge request to review the generated code. You can verify that all your requirements have been met and make any necessary adjustments.\n\n5. **Test the proposed application**: Finally, check that the application runs successfully. With minimal effort on your part, you now have working code that implements your original requirements.\n\n## Improve your development process\n\nGitLab Duo with Amazon Q completely transforms this process, including dramatically decreasing the time it takes to carry out complex developer tasks, through intelligent automation. By leveraging an agentic AI approach, you can accelerate your path from idea to deployment, freeing development teams to focus on more strategic work.\n\nWith GitLab Duo and Amazon Q, you'll develop software faster, more efficiently, and with less manual coding effort. This integration helps you:\n\n* **Save valuable development time** by automating implementation based on requirements  \n* **Maintain consistency** in code generation across your projects  \n* **Reduce the cognitive load** of translating requirements into working code  \n* **Accelerate your release cycles** by removing implementation bottlenecks  \n* **Focus your expertise** on reviewing and optimizing, rather than writing boilerplate code\n\nReady to see GitLab Duo with Amazon Q in action? Watch our demo video to discover how you can transform your development workflow today.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/jxxzNst3jpo?si=j_LQdZhUnwqoQEst\" title=\"GitLab Duo with Amazon Q demo video for dev workflow\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n> To learn more about GitLab Duo with Amazon Q visit us at an upcoming [AWS Summit in a city near you](https://about.gitlab.com/events/aws-summits/) or [reach out to your GitLab representative](https://about.gitlab.com/partners/technology-partners/aws/#form).\n\n## GitLab Duo with Amazon Q resources\n\n- [GitLab Duo with Amazon Q: Agentic AI optimized for AWS generally available](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/)\n- [GitLab and AWS partner page](https://about.gitlab.com/partners/technology-partners/aws/)\n- [GitLab Duo with Amazon Q documentation](https://docs.gitlab.com/user/duo_amazon_q/)",[720,810,9,495,694,232],{"slug":2090,"featured":90,"template":700},"gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes","content:en-us:blog:gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes.yml","Gitlab Duo Amazon Q Transform Ideas Into Code In Minutes","en-us/blog/gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes.yml","en-us/blog/gitlab-duo-amazon-q-transform-ideas-into-code-in-minutes",{"_path":2096,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2097,"content":2103,"config":2108,"_id":2110,"_type":13,"title":2111,"_source":15,"_file":2112,"_stem":2113,"_extension":18},"/en-us/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant",{"title":2098,"description":2099,"ogTitle":2098,"ogDescription":2099,"noIndex":6,"ogImage":2100,"ogUrl":2101,"ogSiteName":685,"ogType":686,"canonicalUrls":2101,"schema":2102},"GitLab Duo Chat 101: Get more done on GitLab with our AI assistant","In this first article in our series learn how Chat can improve developer productivity – for example, by summarizing issues – and how to improve prompts to get better answers faster.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099126/Blog/Hero%20Images/Blog/Hero%20Images/GitLab_Duo_Blog_Hero_1800x945_r2_B%20%281%29_6a2UB7TOQk3JKxyb5yqYtc_1750099126039.png","https://about.gitlab.com/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab Duo Chat 101: Get more done on GitLab with our AI assistant\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2024-05-29\",\n      }",{"title":2098,"description":2099,"authors":2104,"heroImage":2100,"date":2105,"body":2106,"category":718,"tags":2107},[1604],"2024-05-29","GitLab Duo Chat became [generally available](https://about.gitlab.com/blog/gitlab-duo-chat-now-generally-available/) in [GitLab 16.11](https://about.gitlab.com/releases/2024/04/18/gitlab-16-11-released/) and its power as a personal assistant can not be overstated. On a DevSecOps platform, more has to happen than just generating code; planning, discussions, security, compliance, and technical reviews are all critical to developing secure software faster. Issues, epics, merge requests, and other sections of GitLab are where this work happens, with knowledge often buried deep in comment threads. It can take a lot of time to get up to speed on these threads, especially when they've grown to hundreds of comments and interactions and when you've been away from them for a while. This is where GitLab Duo Chat can help.\n\nIn this first part of our GitLab Duo Chat 101 series, we'll introduce you to Chat's capabilities and then dig into how to use Chat to summarize comment threads.\n\n> Live demo! Discover the future of AI-driven software development with our GitLab 17 virtual launch event. [Register today](https://about.gitlab.com/seventeen/)!\n\n## GitLab Duo Chat's capabilities\n\nWith Chat, you can refactor [existing code](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#refactor-code-in-the-ide), learn how a [block of code works](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#explain-code-in-the-ide), and write [tests](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#write-tests-in-the-ide) for your code, learn about your issues and epics, and much more. Depending on your prompts, you can make Chat do impressive things that boost developer productivity. In the video below, I showcased how you can use GitLab Duo Chat to interact with GitLab and learn about your issues and epics.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/RJezT5_V6dI?si=XlXGs2DHAYa8Awzs\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## Use cases   \n\nGitLab Duo Chat’s capabilities allow for productivity gains across multiple parts of the software development lifecycle:\n\n- Product and project managers can use Chat’s issues and epics capabilities to gain insights into discussions and plan faster.\n- Developers can create solutions faster with code suggestions and refactoring capabilities. When it comes to working with legacy code or code from other team members, less time is spent on research with the ` /explain` capability providing the necessary insights to understand the code.\n- Quality assurance and test engineers can generate tests and check for vulnerabilities\n- New employees can get a better understanding of their code base and get started solving problems.\n- Beginner programmers can understand and pick up a language or framework quickly and create solutions with Chat providing next steps and insights.\n\n> Check out \"[10 best practices for using GitLab Duo Chat](https://about.gitlab.com/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat/)\" for tips and tricks to craft AI prompts. \n\n## Summarizing issues\n\nWhen you encounter an issue, especially one with a lot of comments, you skim through the issue description, along with a couple of comments, but can't always get the complete picture of the conversations. GitLab Duo Chat can get you up to speed fast. In the image below, I asked Chat to summarize an issue along with a follow-up question. In two prompts, I got what I needed to understand what is going on in the issue without spending hours reading through the comments.\n\n![Chat summarizing an issue](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099137154.png)\n\nYou can use GitLab Duo Chat on the GitLab interface, as well as [the WebIDE, Visual Studio Code, and JetBrains interfaces](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#use-gitlab-duo-chat-in-the-web-ide).\n\n## Prompts and context\n\nGetting the best responses from any AI tool requires carefully crafting the questions asked. Sometimes, you need to give examples of the responses you expect to prime the large language models (LLMs) toward a desired response. Here are some areas to focus on to get desired responses.\n\n### Context\n\nHere are three prompts with similar objectives but worded differently:\n\n| Prompt 1     | Prompt 2     | Prompt 3    |\n| ---------- | ---------- | ---------- |\n| ![Prompt: Can you summarize this issue's description?](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099137154.png)       | ![Prompt: Can you provide a high-level summary of this issue?](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099137155.png)      | ![Prompt: Why is this issue popular?](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099137156.png)      |\n\nThe context, “this issue,” is common among the three prompts; this tells Chat what resource to use in looking for answers. Prompt 1 gives additional context on what to focus on: the description of the issue. Prompt 2 is not limited in its scope, which means the LLMs will spend more time going through the description and all the comments to provide a more detailed summary of the whole issue. (Note: As of the publication of this blog, there were more than 90 comments in that issue.) Prompt 3 got a poorer response because not much expectation was set for the type of response expected. \n\n[Low-context communication](https://handbook.gitlab.com/handbook/company/culture/all-remote/effective-communication/#understanding-low-context-communication) is critical in crafting your prompt for the best responses, as all information needed for the LLMs to provide an informed response is provided.\n\n### Simplicity\n\nThe wordiness of prompts can sometimes lead to incorrect or no responses. In the image below, you can see that rephrasing a prompt from “Customers have mentioned why this issue is important to them. Can you list the top 3 reasons they mentioned?” to “Why is this issue important to customers?” led to the expected response. When you don’t get the response you desire, simplifying or changing the words used in your prompt can improve the quality of responses.\n\n![Wordy Chat prompts](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099137158.png)\n\n### Follow-up questions\n\nGitLab Duo Chat can have follow-up conversations – an essential capability. In the image below, I continued asking how the issue in question can be solved in GitLab's code along with a follow-up question asking for code samples.\n\n![Streamlined Chat prompt shown](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099137/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099137158.png)\n\nFollow-up questions allow the application to maintain context and provide faster responses. A recommendation is to provide parts of Chat’s previous responses in the next prompt. In the example above, I mentioned “Rails App,” as previously suggested. \n\n## Get started with GitLab Duo Chat \n\nGitLab Duo Chat does more than help you write better code, it helps you navigate through problems and quickly find solutions. With the right prompts and context, you can build secure software faster.\n\n> Want to try GitLab Duo Chat? [Start your free trial](https://about.gitlab.com/gitlab-duo/#free-trial) today.",[720,495,851,9],{"slug":2109,"featured":90,"template":700},"gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant","content:en-us:blog:gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant.yml","Gitlab Duo Chat 101 Get More Done On Gitlab With Our Ai Assistant","en-us/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant.yml","en-us/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant",{"_path":2115,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2116,"content":2122,"config":2127,"_id":2129,"_type":13,"title":2130,"_source":15,"_file":2131,"_stem":2132,"_extension":18},"/en-us/blog/gitlab-duo-chat-gets-agentic-ai-makeover",{"title":2117,"description":2118,"ogTitle":2117,"ogDescription":2118,"noIndex":6,"ogImage":2119,"ogUrl":2120,"ogSiteName":685,"ogType":686,"canonicalUrls":2120,"schema":2121},"GitLab Duo Chat gets agentic AI makeover  ","Our new Duo Chat experience, currently an experimental release, helps developers onboard to projects, understand assignments, implement changes, and more.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png","https://about.gitlab.com/blog/gitlab-duo-chat-gets-agentic-ai-makeover","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab Duo Chat gets agentic AI makeover  \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Torsten Linz\"}],\n        \"datePublished\": \"2025-05-29\",\n      }",{"title":2117,"description":2118,"authors":2123,"heroImage":2119,"date":2124,"body":2125,"category":718,"tags":2126},[1343],"2025-05-29","Generative AI chat assistants have become standard in software development,\nhelping create and fix code just to start. But what if your chat assistant\ncould understand the artifacts of your entire development process, not just\nyour code? What if that chat assistant could help you work through issues\nand project documentation before it helps you write code, and could access\nCI/CD pipelines and merge requests to help you finish coding tasks\nproperly? \n\n\n**Meet the next generation of GitLab Duo Chat – GitLab Duo Agentic Chat, a\nsignificant evolution in AI-native development assistance and the newest\naddition to our platform, now in [experimental\nrelease](https://docs.gitlab.com/policy/development_stages_support/#experiment).**\nGitLab Duo Agentic Chat is currently available as an experimental feature in\nVS Code to all users on GitLab.com that have any one of these add-ons:\nDuo Core, Duo Pro, or Duo Enterprise.\n\n\nAgentic Chat transforms chat from traditional conversational AI to a chat\nexperience that takes action on your behalf, breaking down complex problems\ninto discrete tasks that it can complete. Instead of simply responding to\nquestions with the context you provide, Agentic Chat can:\n\n\n* **Autonomously determine** what information it needs to answer your\nquestions  \n\n* **Execute a sequence of operations** to gather that information from\nmultiple sources  \n\n* **Formulate comprehensive responses** by combining insights from across\nyour project  \n\n* **Create and modify files** to help you implement solutions\n\n\nAnd all of this is done while keeping the human developer within the loop.\n\n\nAgentic Chat is built on the Duo Workflow architecture, which is [currently\nin private\nbeta](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/).\nThe architecture comprises agents and tools that take on specific tasks like\nfinding the right context for a given question or editing files. \n\n\n**Use cases for GitLab Duo Agentic Chat**\n\n\nHere are some real-world and common use cases for Agentic Chat:\n\n\n* Onboard to new projects faster by having AI help you familiarize yourself\nwith a new codebase.\n\n\n* Jump into assigned work immediately, even when issue descriptions are\nunclear, because Agentic Chat can help you connect the dots between\nrequirements and existing implementations.\n\n\n* When it's time to make changes, Agentic Chat can handle the implementation\nwork by creating and editing multiple files across your project.\n\n\n* At release time, Agentic Chat can help you verify that your solution\nactually addresses the original requirements by analyzing your merge\nrequests against the initial issue or task.\n\n\n![agentic chat -\nexample](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099210/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099210429.png)\n\n\n\u003Ccenter>\u003Ci>Agentic Chat making code edits\u003C/i>\u003C/center>\n\n\n## From learning to shipping: A complete workflow demonstration in four\nsteps\n\n\nTo show how Agentic Chat transforms the development experience, let's walk\nthrough a real scenario from our engineering teams. Imagine you're a new\nteam member who's been assigned an issue but knows nothing about the\ncodebase. You can follow along with this video demonstration:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/uG9-QLAJrrg?si=kaOhYylMIaWkIuG8j\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n**Step 1: Understand the project**\n\n\nInstead of manually exploring files and documentation, you can prompt\nAgentic Chat:\n\n\n```unset\n\nI am new to this project. Could you read the project structure and explain\nit to me?\n\n```\n\n\nAgentic Chat provides a comprehensive project overview by:  \n\n- Exploring the directory structure  \n\n- Reading README files and documentation  \n\n- Identifying key components and applications\n\n\n**Step 2: Understand your assigned task**\n\n\nNext, you need to understand your specific assignment, so you can enter this\nprompt:\n\n\n```unset\n\nI have been assigned Issue 1119. Could you help me understand this task,\nspecifically where do I need to apply the refactoring?\n\n```\n\n\nAgentic Chat explains the task and proposes a refactoring approach by:\n\n- Retrieving and analyzing the issue details from the remote GitLab server  \n\n- Examining relevant project files  \n\n- Identifying the specific locations requiring changes\n\n\n**Step 3: Implement the solution**\n\n\nRather than doing the work manually, you can request:\n\n\n```unset\n\nCould you make the edits for me? Please start with steps one, two, three.\n\n```\n\n\nAgentic Chat then:  \n\n- Creates new directories and files as needed \n\n- Extracts and refactors code across multiple locations  \n\n- Ensures consistency across all modified files  \n\n- Provides a summary of all changes made\n\n\n**Step 4: Verify completion**\n\n\nFinally, after creating your merge request, you can verify your work:\n\n\n```unset\n\nDoes my MR fully address Issue 1119? \n\n```\n\n\nAgentic Chat confirms whether all requirements have been met by analyzing\nboth your merge request and the original issue.\n\n\n## Try it today and share your feedback\n\n\nGitLab Duo Agentic Chat is currently available as an experimental feature in VS Code to all users on GitLab.com that have any one of these add-ons: Duo Core, Duo Pro, or Duo Enterprise. See our [setup\ndocumentation](https://docs.gitlab.com/user/gitlab_duo_chat/agentic_chat/)\nfor prerequisites and configuration steps.\n\n\nAs an experimental feature, Agentic Chat has some known limitations we're\nactively addressing, including slower response times due to multiple API\ncalls, keyword-based rather than semantic search, and limited support for\nnew local folders or non-GitLab projects. **Your feedback is crucial in\nhelping us prioritize improvements and bring Agentic Chat to general\navailability so please share your experience in [this\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/542198).**\n\n\n## What's next?\n\n\nWe are fully focused on improving Agentic Chat, including bringing it to\ngeneral availability. In the meantime, we are aiming to improve response\ntimes and are adding capabilities that GitLab Duo Chat currently has, such\nas using self-hosted models or supporting JetBrains and Visual Studio in\naddition to VS Code. Once we have switched Duo Chat to this new architecture\nwe plan to also bring Agentic Chat to the chat in the GitLab web\napplication. We also plan to add a lot more functionality, such as editing\nGitLab artifacts, supporting context from custom Model Context Protocol, or\nMCP, servers, and offering commands to run in the terminal.\n\n\n> Ready to experience autonomous development assistance but not yet a GitLab\ncustomer? Try Agentic Chat today as part of [a free trial of GitLab\nUltimate with Duo Enterprise](https://about.gitlab.com/free-trial/) and help\nshape the future of AI-powered development. Follow these [setup steps for VS\nCode](https://docs.gitlab.com/user/gitlab_duo_chat/agentic_chat/#use-agentic-chat-in-vs-code).\n\n>\n\n> And make sure to join the GitLab 18 virtual launch event to learn about\nour agentic AI plans and more. [Register\ntoday!](https://about.gitlab.com/eighteen/)\n\n\n***Disclaimer: This blog contains information related to upcoming products,\nfeatures, and functionality. It is important to note that the information in\nthis blog post is for informational purposes only. Please do not rely on\nthis information for purchasing or planning purposes. As with all projects,\nthe items mentioned in this blog and linked pages are subject to change or\ndelay. The development, release, and timing of any products, features, or\nfunctionality remain at the sole discretion of GitLab.***\n\n\n## Learn more\n\n\n- [GitLab Duo Workflow: Enterprise visibility and control for agentic\nAI](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/)\n\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n\n- [Agentic AI guides and\nresources](https://about.gitlab.com/blog/agentic-ai-guides-and-resources/)\n",[720,891,696,495,694,9],{"slug":2128,"featured":90,"template":700},"gitlab-duo-chat-gets-agentic-ai-makeover","content:en-us:blog:gitlab-duo-chat-gets-agentic-ai-makeover.yml","Gitlab Duo Chat Gets Agentic Ai Makeover","en-us/blog/gitlab-duo-chat-gets-agentic-ai-makeover.yml","en-us/blog/gitlab-duo-chat-gets-agentic-ai-makeover",{"_path":2134,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2135,"content":2140,"config":2146,"_id":2148,"_type":13,"title":2149,"_source":15,"_file":2150,"_stem":2151,"_extension":18},"/en-us/blog/gitlab-gdk-remote-development",{"title":2136,"description":2137,"ogTitle":2136,"ogDescription":2137,"noIndex":6,"ogImage":1875,"ogUrl":2138,"ogSiteName":685,"ogType":686,"canonicalUrls":2138,"schema":2139},"Contributor how-to: Remote Development workspaces and GitLab Developer Kit","This tutorial helps you get GDK working inside Remote Development workspaces to begin contributing to GitLab.","https://about.gitlab.com/blog/gitlab-gdk-remote-development","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Contributor how-to: Remote Development workspaces and GitLab Developer Kit\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Raimund Hook\"}],\n        \"datePublished\": \"2023-07-31\",\n      }",{"title":2136,"description":2137,"authors":2141,"heroImage":1875,"date":2143,"body":2144,"category":785,"tags":2145},[2142],"Raimund Hook","2023-07-31","Open source is fundamental to GitLab. We believe that [everyone can\ncontribute](https://about.gitlab.com/company/mission/#mission).\n\nTypically, we recommend that anyone contributing anything more than basic\nchanges to GitLab run the [GitLab Development\nKit](https://gitlab.com/gitlab-org/gitlab-development-kit) (GDK). Because\ncontributors can't always meet the GDK's resource demands, we're working to\nenable GDK inside the cloud-based GitLab Remote Development workspaces.\n\n\nIn this article, I'll explain how I used a Remote Development workspace\nrunning in my Kubernetes cluster to make working with the GDK faster and\neasier.\n\n\n## A preliminary note\n\nFirst, keep in mind that as of this writing the [Remote Development\nworkspaces](https://about.gitlab.com/direction/create/ide/remote_development/)\nfeature is still in Beta. My example here is therefore very much a proof of\nconcept — and as such, it has some rough edges.\n\n\nBefore getting started, I followed the \"[Set up a\nworkspace](https://docs.gitlab.com/ee/user/workspace/#set-up-a-workspace)\"\nprerequisites guide in the GitLab docs. For a more detailed set of\ninstructions, see Senior Developer Evangelist Michael Friedrich's tutorial\non [how to set up infrastructure for cloud development\nenvironments](https://about.gitlab.com/blog/set-up-infrastructure-for-cloud-development-environments/).\n\n\n## Getting started with workspaces\n\nTo start using workspaces, you will need a project configured with a\n`.devfile.yaml`. GitLab team members have curated [a number of example\nprojects](https://gitlab.com/gitlab-org/remote-development/examples) you can\nreview.\n\n\nInitially, I tried to do this with a fork of the GitLab project itself, but\nI ran into [some\nissues](https://gitlab.com/gitlab-org/gitlab/-/issues/414011) when the\nworkspace begins cloning the repository.\n\n\nTo figure out what was causing my problems, I looked more closely at what\nhappens behind the scenes when a workspace is created.\n\n\n## Behind the scenes with Remote Development workspaces\n\nWhen you create a new workspace, the following happens:\n\n1. The GitLab agent for Kubernetes creates a new namespace in your cluster.\nThe agent dynamically generates a name for and assumes management of the\nnamespace.\n\n1. Inside the namespace, a new deployment is created, specifying the\ncontainer you chose in your `.devfile.yaml` as the image to use.\n\n1. This deployment is configured with some [init\ncontainers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/)\nthat perform some actions:\n    1. Cloning the repository into `/project/${project_path}`.\n    1. Injecting the VS Code server binary into your container.\n1. Once those init containers are complete, your container starts and the\nworkspace becomes available.\n\n\n## The clone problem\n\nWhen cloning a repository, `git` tends to do much of the work in memory.\nThis can be a challenge on larger projects/repositories, as it can require\nsignificant amounts of RAM. When cloning the GitLab project, for instance,\ngit consumes approximately 1.6GB of RAM. This number is only going to\nincrease with time. Sure, strategies like [shallow\nclones](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt)\ncan help reduce this, but these are perhaps less suited to active use by a\ndeveloper as they can increase the amount of time required to perform\nongoing git operations.\n\n\nIn fact, creating a workspace using our `.devfile.yaml` in a fork of the\nGitLab project failed for this reason. The init container performing the\nclone is currently hard-limited to 128MiB of RAM, after which the memory\nmanagement processes on the node kill the container.\n\n\nTo overcome this limitation, move the `.devfile.yaml` into the a fork of the\nroot of the GDK repository. This project clones more quickly (and does so\nusing fewer resources), so it's a  perfect starting point for running GDK\nitself. Another (bonus) advantage: You're then primed to contribute to the\nGDK itself, in addition to any of the other GitLab projects that the GDK\nclones.\n\n\n## Components of a GDK installation\n\nGDK clones the following projects from the GitLab 'family':\n\n* [GitLab](https://gitlab.com/gitlab-org/gitlab)\n\n* [Gitaly](https://gitlab.com/gitlab-org/gitaly)\n\n* [GitLab shell](https://gitlab.com/gitlab-org/gitlab-shell)\n\n\nThis allows you to work on any items in those directories as a part of your\n\"live\" installation.\n\n\n## Getting GDK installed and running in a workspace\n\nOnce I had a workspace up and running, my next step was to get GDK installed\nand running *in* that workspace. The GDK's documentation presents [several\nroutes for doing\nthis](https://gitlab.com/gitlab-org/gitlab-development-kit/#installation).\n\n\nA complete installation can take some time, as GDK needs to bootstrap itself\nand install a number of prerequisites. This is less than ideal in the\ncontext of a Remote Development workspace, as one of remote development's\nprimary benefits is enabling access to a development environment rapidly.\nRequiring a user to bootstrap an environment that takes 50 minutes (or\nlonger) doesn't help achieve this goal.\n\n\nTo combat this, I built a container image that effectively bootstraps and\ninstalls GDK, pre-building the GDK prerequisites and pre-seeding the\ndatabase. This image and its associated tooling are currently [in\nreview](https://gitlab.com/gitlab-org/gitlab-development-kit/-/merge_requests/3231).\n\n\n## Pre-building\n\nPre-building the container and running the bootstrap process on a scheduled\nbasis allows us to perform that process once, without requiring the user to\nwait for something that can essentially be \"pre-canned\" for their use.\n\n\nOnce the workspace is running, we still need to \"reinstall\" the GDK\nenvironment with the latest version of our GitLab repository, but this step\ndoesn't take quite as long as a complete bootstrap.\n\n\n## Generating a gdk.yml file\n\nTo work properly, GDK also requires a [`gdk.yml`\nfile](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/configuration.md#gdkyml).\nThis file tells GDK how to configure GitLab to return the correct URLs and\nother items. To get GDK running in Remote Development, Rails needs to return\nURLs in a certain scheme (otherwise your browser won't know where to\nconnect). To help this along, we [inject an environment\nvariable](https://gitlab.com/gitlab-org/gitlab/-/issues/415328) into the\nworkspace container. This variable helps us determine the URL in use (which\nis dynamically generated for each workspace).\n\n\nWe [now have a\nscript](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/support/gitlab-remote-dev/remote-dev-gdk-bootstrap.sh?ref_type=heads)\nin GDK that will generate your `gdk.yml` file based on your workspace.\n\n\n## Creating our devfile\n\nThe contents of my `.devfile.yaml` looks like this:\n\n\n```yaml\n\nschemaVersion: 2.2.0\n\ncomponents:\n  - name: tooling-container\n    attributes:\n      gl/inject-editor: true\n    container:\n      # NB! This image is only in use until https://gitlab.com/gitlab-org/gitlab-development-kit/-/merge_requests/3231 is merged!\n      image: registry.gitlab.com/gitlab-org/gitlab-development-kit/gitlab-remote-workspace:stingrayza-gdk-remote-dev-add-container\n      memoryRequest: 10240M\n      memoryLimit: 16384M\n      cpuRequest: 2000m\n      cpuLimit: 6000m\n      endpoints:\n        - name: ssh-2222\n          targetPort: 2222\n        - name: gdk-3000\n          targetPort: 3000\n        - name: docs-3005\n          targetPort: 3005\n        - name: pages-3010\n          targetPort: 3010\n        - name: webpack-3808\n          targetPort: 3808\n        - name: devops-5000\n          targetPort: 5000\n        - name: jaeger-5778\n          targetPort: 5778\n        - name: objects-9000\n          targetPort: 9000\n        - name: shell-9122\n          targetPort: 9122\n```\n\n\nThis definition comes straight out of the [Workspace\ndocs](https://docs.gitlab.com/ee/user/workspace/#devfile), and opens a\nnumber of ports that GDK uses. (For now, I've only tested the port\n`gdk-3000`, which is the the link to our instance of GDK.)\n\n\n## From Workspace to GDK\n\nOnce we have a project with a `.devfile.yaml`, our final step is to [create\na new\nworkspace](https://docs.gitlab.com/ee/user/workspace/#create-a-workspace).\n\n\nAs a part of this step, your cluster will pull the image as defined in the\n`.devfile.yaml` and start it up. For the GDK image we pre-built, this can\ntake a few minutes.\n\n\nOnce the workspace is ready, the last step is to follow the link from the UI\nto connect to the workspace. This will open up a familiar VS Code IDE, with\nour GDK fork checked out.\n\n\nBut wait, where's GDK?\n\n\nWell, the pre-build did most of the work for us, but we still need to take a\nfew final steps before we can claim that GDK is up and running. These have\nbeen built into a script we can run from the integrated terminal within the\nworkspace.\n\n\nTo open a terminal, we can click on the VS Code Hamburger menu (top left),\nnavigate to `Terminal` and select `New Terminal`.\n\n\nNow we execute the following script, which completes the setup and copies a\ncouple of files over from the pre-built folders:\n\n\n```shell\n\nsupport/gitlab-remote-dev/remote-dev-gdk-bootstrap.sh\n\n```\n\n\nThis can take up to 15 minutes, but when it's done it should output the\nmagic words — something like the following (note the 3000 in the URL; we\nspecified that in the `.devfile.yaml` earlier):\n\n\n```shell\n\nSuccess! You can access your GDK here:\nhttps://3000-workspace-62637-2083197-apglwp.workspace.my-workspace.example.net/\n\n```\n\n\n## Connect to your GDK\n\nFollow the link as displayed using Cmd-click or Ctrl-click. After a couple\nof moments (GDK boot time), you should reach a familiar GitLab login screen.\n\n\nCongratulations! GDK is now running inside your Remote Development\nworkspace.\n\n\nTo log in, type `gdk` in your terminal and you'll see the default admin\ncredentials displayed near the bottom:\n\n\n```shell\n\n# Development admin account: xxxx / xxxx\n\n\nFor more information about GitLab development see\n\nhttps://docs.gitlab.com/ee/development/index.html.\n\n```\n\n\nLog into your GDK with the default credentials, change the admin user\npassword, and you're all set!\n\n\n## Demo of workspace launch\n\nHere's a demo of launching a workspace in my personal cluster:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/iXq1NnTjnX0\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## How to contribute to GitLab\n\nIn this article I explained how to get GDK up and running in Remote\nDevelopment workspaces. This is not without its challenges, but the end\nresult should mean that contributing to GitLab (especially in\nresource-constrained environments) is quicker and easier.\n\n\nDo you want to contribute to GitLab? Come and join in the conversation in\nthe `#contribute` channel on GitLab's [Discord](https://discord.gg/gitlab),\nor just pop in and say \"hello.\"\n\n\n_Disclaimer: This blog contains information related to upcoming products,\nfeatures, and functionality. It is important to note that the information in\nthis blog post is for informational purposes only. Please do not rely on\nthis information for purchasing or planning purposes. As with all projects,\nthe items mentioned in this blog and linked pages are subject to change or\ndelay. The development, release, and timing of any products, features, or\nfunctionality remain at the sole discretion of GitLab._\n",[9,789,956,1898,1306,743],{"slug":2147,"featured":6,"template":700},"gitlab-gdk-remote-development","content:en-us:blog:gitlab-gdk-remote-development.yml","Gitlab Gdk Remote Development","en-us/blog/gitlab-gdk-remote-development.yml","en-us/blog/gitlab-gdk-remote-development",{"_path":2153,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2154,"content":2160,"config":2165,"_id":2167,"_type":13,"title":2168,"_source":15,"_file":2169,"_stem":2170,"_extension":18},"/en-us/blog/gitlab-importers",{"title":2155,"description":2156,"ogTitle":2155,"ogDescription":2156,"noIndex":6,"ogImage":2157,"ogUrl":2158,"ogSiteName":685,"ogType":686,"canonicalUrls":2158,"schema":2159},"How to migrate data to GitLab using main importers","Learn about the capabilities of main importers, which are used to import data from external tools and from other GitLab instances.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679170/Blog/Hero%20Images/migration-data.jpg","https://about.gitlab.com/blog/gitlab-importers","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to migrate data to GitLab using main importers\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2023-02-13\",\n      }",{"title":2155,"description":2156,"authors":2161,"heroImage":2157,"date":2162,"body":2163,"category":785,"tags":2164},[1096],"2023-02-13","\n\nA typical organization looking to adopt GitLab already uses many other tools. Artifacts such as code, build pipelines, issues, and epics will already exist and be changed daily. A seamless transition of work in progress is, therefore, critically important when importing data. GitLab importers aim to make this process easy and reliable, ensuring data is imported quickly and with maximum care.\n\nAt GitLab, a dedicated development team, named group:import, creates a seamless experience when importing data into GitLab or from one GitLab instance to another. This team continuously develops and improves the importing experience and keeps our importers up to date with new features and capabilities.\n\n## Migrate groups by direct transfer\n\nUsing group migration, you can import groups from one GitLab instance to another instance. The most common use case is to import groups from self-managed GitLab instances to GitLab.com (GitLab SaaS). With the group migration, you can migrate many groups in a single click.\n\n### Which items are imported?\n\nThe group migration imports the entire group structure, including all the sub groups and projects in them. Currently, to import projects as part of the group migration on self-managed GitLab, the administrator needs to enable the feature flag named `bulk_import_projects`. On GitLab.com, our SaaS offering, migration of both groups and projects is available. More information can be found in our [documentation](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended).\n\nThe team continuously adds objects to the migration, but not all group items are imported. The docs cover the [items that are imported](https://docs.gitlab.com/ee/user/group/import/#migrated-group-items). \n\n### How can groups be imported?\n\nIt is very simple to import groups between two instances. Here are the steps: \n\n- Create a new group or subgroup in the designated instance \n- Select \"Import group\" \n- Connect to the remote instance with your [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)\n- Select the source groups you want to import \n- Click \"Import xyz groups\"\n\n![bulk_imports_v14_1](https://about.gitlab.com/images/blogimages/2022-11-15-gitlab-importers/bulk_imports_v14_1.png)\n\n## File-based import/export (the previously used method)\n\nGroup migration is the preferred method to migrate content from one GitLab instance to another, as it automates the process and you can import many groups in a single click. However, for some use cases, such as air-gapped networks when you don't have network connection between the two instances, or when you have environments with limited connectivity, the group migration won't help because it requires connection between the two instances. File-based export/import for [groups](https://docs.gitlab.com/ee/user/group/settings/import_export.html) and [projects](https://docs.gitlab.com/ee/user/project/settings/import_export.html) can be used when there is no connectivity between the instances. \n\nFile-based export/import is a manual process and requires a few steps in order to migrate each group or project. The file-based import/export is available from the UI and in the API. The team plans to disable it by a feature flag soon to encourage users to use group migration. However, you will be able to enable the feature flag in your instance if your use case requires the file-based import/export. More info can be found in this [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/363406).\n\n## Import projects from external tools  \n\nGitLab has built-in support for import projects from [a variety of tools](https://docs.gitlab.com/ee/user/project/import/).\n\nThe GitHub importer is the most common importer and, therefore, the team invests a lot of effort to add more migrated components. GitLab and GitHub have different structure and architecture, so sometimes it is tricky to import objects from GitHub when the migrated components are implemented differently in GitLab. So the team needs to find creative ways to map some of the features or configurations. This is an example [epic](https://gitlab.com/groups/gitlab-org/-/epics/8585 ) with a proposal to map rules for protected branches when migrating GitHub protected rules. \n\n\n### What can be imported from GitHub to GitLab?\n\n- Repository description\n- Git repository data\n- Branch protection rules\n- Issues\n- Pull requests\n- Wiki pages\n- Milestones\n- Labels\n- Pull request review comments\n- Regular issue and pull request comments\n- Attachments for\n    - Release notes\n    - Comments and notes\n    - Issue description\n    - Merge Request description\n- Git Large File Storage (LFS) objects\n- Pull request reviews \n- Pull request “merged by” information \n- Pull request comments replies in discussions \n- Diff notes suggestions \n- Release note descriptions\n\nHere is a [full list of imported data](https://docs.gitlab.com/ee/user/project/import/github.html#imported-data).\n\nRead what's next in our [GitHub Epic](https://gitlab.com/groups/gitlab-org/-/epics/2984). \n\n### Repository by URL\n\nAn alternative way to import external projects is the Repository by URL option. You can import any Git repository through HTTP from the *Import Project* page, by choosing \"Repository by URL\".\n\nTo learn more about the Importer direction, roadmap, etc., refer to [Category Direction - Importers](/direction/manage/import_and_integrate/importers/).\n\n_Cover image by [Conny Schneider](https://unsplash.com/@choys_?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyTex) on [Unsplash](https://unsplash.com/s/photos/data-migration?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)_\n",[9,789,744],{"slug":2166,"featured":6,"template":700},"gitlab-importers","content:en-us:blog:gitlab-importers.yml","Gitlab Importers","en-us/blog/gitlab-importers.yml","en-us/blog/gitlab-importers",{"_path":2172,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2173,"content":2179,"config":2186,"_id":2188,"_type":13,"title":2189,"_source":15,"_file":2190,"_stem":2191,"_extension":18},"/en-us/blog/gitlab-instance-security-best-practices",{"title":2174,"description":2175,"ogTitle":2174,"ogDescription":2175,"noIndex":6,"ogImage":2176,"ogUrl":2177,"ogSiteName":685,"ogType":686,"canonicalUrls":2177,"schema":2178},"GitLab instance: security best practices","Default settings on products can be massively helpful. However, when it comes to hardening your GitLab instance, we’ve got some helpful configuration recommendations from our security team.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667057/Blog/Hero%20Images/configs_unsplash.jpg","https://about.gitlab.com/blog/gitlab-instance-security-best-practices","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab instance: security best practices\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Mark Loveless\"}],\n        \"datePublished\": \"2020-05-20\",\n      }",{"title":2174,"description":2175,"authors":2180,"heroImage":2176,"date":2182,"body":2183,"category":830,"tags":2184},[2181],"Mark Loveless","2020-05-20","GitLab is a feature-rich and powerful collaboration tool that is easy to use, and our self-managed installation is intended to be ready-to-go right out of the box. Exposing *any* service to the internet can create its own challenges from a security perspective, and as a result an administrator might have a bit of head-scratching over how to set things up safely.\n\nFortunately, we have a large number of security features and options that can be used to help lock things down. In this blog post, we’ve highlighted a few important features that will certainly help an administrator harden that new GitLab instance - particularly one facing the internet.\n\n## Access basics\n\nDuring the initial GitLab installation, you will be asked to set up a root password. Obviously, we highly recommend a long password, unique to your GitLab instance that is not easily guessable with a mixture of uppercase and lowercase along with numbers and special characters. For a working example, see how we advise GitLab team members to create, store and manage [passwords](https://handbook.gitlab.com/handbook/security/#accounts-and-passwords).\n\nTo help simplify your installation, consider using environment variables. The root password can also be set this way. For example:\n\n`GITLAB_ROOT_PASSWORD=hunter2 GITLAB_HOST=https://hunter2.instance apt install gitlab-ee`\n\nThis has the added advantage of kicking off the entire [letsencrypt](https://letsencrypt.org/) process to ensure up-to-date certificates are used for your instance.\n\nYou will also want to ensure that users of your instance are also using strong, unique passwords, and you will want to ensure that the methods they use to access your instance are solid. Again, refer to our documentation on passwords for some ideas to share.\n\nThere are some choices you can make to limit access to data and restrict access to authorized users. In **Admin Area > Settings > General** you will want to expand the \"visibility and access controls\" section and make a few changes.\n\nTo help secure SSH access, RSA SSH keys should be allowed, as well as ED25519. Without going *too* deep, the open source crowd seems to prefer ED25519 as everything about it is open source (well-documented, trustworthy elliptical curve parameters), whereas other algorithms do not specify or go into details as to why they chose certain values. DSA also has a theoretical attack that could be used against it, although RSA could in theory fall to the same attack but is more resistant. Ah, but I digress! The main reason to support both RSA and ED25519 is that older systems that will connect may not be set up for ED25519, but will still support RSA, so at least both are recommended. With respect to RSA, encourage your users to use 2048 bits or higher when configuring keys.\n\nWe highly recommend using passwordless SSH authentication over password authentication. The communications are more secure (passwordless SSH authentication uses public/private key cryptography), it allows for an easier workflow, and it is one less password to worry about.\n\nFor more on SSH keys, see our documentation on [ssh keys restrictions](https://docs.gitlab.com/ee/security/ssh_keys_restrictions.html), as well as the additional [visibility and access control](https://docs.gitlab.com/ee/administration/settings/visibility_and_access_controls.html) settings that can be configured.\n\n## Restricting how and who\n\nThere are a few settings we recommend tweaking to help define how users access our instance and who we even allow to have access. You’ll want to check out three areas in particular under the **Admin Area > Settings > General** settings.\n\n**Sign up restrictions:**\n* Ensure open sign-up is disabled on your instance. Open registration is disabled by default on self-managed instances with GitLab 13.6 and above installed. If new sign-up is enabled and your instance is open to the internet, anyone can sign up and access data. Administrators who would like to further restrict access on their instance can [follow our documentation on how to configure user access](https://docs.gitlab.com/ee/administration/settings/sign_up_restrictions.html#disable-new-sign-ups).\n* Make sure that Send confirmation email on sign-up\" is checked. This adds a level of assurance that the user is in fact a real user.\n* If you want to restrict access to a sub-group such as the users in your organization, consider configuring a whitelist for your organization’s domain, (e.g., \"example.com\") which will allow them to sign up.\n* Minimum password length: 12. For users that are allowed access, make sure they will be using longer passwords. See our [password length limits documentation](https://docs.gitlab.com/ee/security/password_length_limits.html) for details.\n* For more detailed information, see our [documentation around sign up restrictions](https://docs.gitlab.com/ee/administration/settings/sign_up_restrictions.html)\n\n**Sign in restrictions:**\n* Make sure that Require 2FA is enabled. Multifactor authentication is the more secure method of protecting authentication to a user's account, and is strongly encouraged.\n* Disable \"password authentication enabled for Git over HTTP(S)\" if for some reason you can’t require MFA. This will require users to use a personal access token, further securing the user accounts.\n* For more detailed information, check our [documentation around sign in restrictions](https://docs.gitlab.com/ee/administration/settings/sign_in_restrictions.html).\n\n**Visibility and privacy:**\nEnsure project visibility is set to [\"Private\"](https://docs.gitlab.com/ee/user/public_access.html) on [existing projects](https://docs.gitlab.com/ee/user/public_access.html) and [by default for *new* projects](https://docs.gitlab.com/ee/administration/settings/visibility_and_access_controls.html#default-project-visibility). Private projects can only be cloned, downloaded, or viewed by project members, newly registered users will not be able to access these projects.\n\n## Improving performance and network tweaks\n\nThere are a few settings that will allow you to help protect your system from various network usage spikes, making your system a lot more stable and accessible for users.\n\n#### User and IP rate limits\nGoing to **Admin Area > Network > User and IP rate limits** allows you to make a few adjustments. Specifically you will want all three items checked:\n\n* \"Enable unauthenticated request rate limit\"\n* \"Enable authenticated API request rate limit\"\n* \"Enable authenticated web request rate limit\"\n\nThe default values associated with those items should be fine under most conditions. For more information, see our [documentation around user and IP rate limits](https://docs.gitlab.com/ee/administration/settings/user_and_ip_rate_limits.html).\n\n#### Webhooks\nWebhooks are a useful feature with a lot of power. Unless there is a legitimate need to allow webhooks to communicate with internal services, they should be restricted to services that are publicly reachable, which you can verify in **Admin Area > Network > Outbound Requests**. While the \"allow requests to the local network from web hooks and services\" is disabled by default, you should also uncheck \"allow requests to the local network from system hooks\" as well. For more detail, including some of the dangers inherent in webhooks, see our [webhooks documentation](https://docs.gitlab.com/ee/security/webhooks.html).\n\n#### Protected paths\nIn **Admin Area > Network > Protected Paths** ensure that \"Enable protected paths rate limit\" has been checked. Default values should be more than sufficient. For details, check out our [protected paths documentation](https://docs.gitlab.com/ee/administration/settings/protected_paths.html).\n\n## Customize your configuration, harden your instance\n\nWe understand with security there is always a balance between protection and agility. In the cases of customers with internet-facing GitLab instances, there are often choices driven by a combination of different business drivers and needs. However, with the help of a few configuration tweaks you can harden your instance and better protect your organization, while still remaining open to the internet.\n\nAdditional settings, including those with security implications, can be found in the [Admin Area](https://docs.gitlab.com/ee/administration/settings/). You'll want to explore those to really fine-tune your setup and make it your own. For some of you, these will have their own security implications that may be unique to your organization. Have fun exploring and securing your instance!\n\nCover image by [Alexey Ruban](https://unsplash.com/@intelligenciya) on [Unsplash](https://unsplash.com/)\n{: .note}\n",[830,2185,9],"security research",{"slug":2187,"featured":6,"template":700},"gitlab-instance-security-best-practices","content:en-us:blog:gitlab-instance-security-best-practices.yml","Gitlab Instance Security Best Practices","en-us/blog/gitlab-instance-security-best-practices.yml","en-us/blog/gitlab-instance-security-best-practices",{"_path":2193,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2194,"content":2200,"config":2206,"_id":2208,"_type":13,"title":2209,"_source":15,"_file":2210,"_stem":2211,"_extension":18},"/en-us/blog/gitlab-mitre-attack-navigator",{"title":2195,"description":2196,"ogTitle":2195,"ogDescription":2196,"noIndex":6,"ogImage":2197,"ogUrl":2198,"ogSiteName":685,"ogType":686,"canonicalUrls":2198,"schema":2199},"Use GitLab and MITRE ATT&CK Navigator to visualize adversary techniques","This tutorial helps build and deploy a customized version of MITRE's ATT&CK Navigator using GitLab CI/CD and GitLab Pages.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749665667/Blog/Hero%20Images/built-in-security.jpg","https://about.gitlab.com/blog/gitlab-mitre-attack-navigator","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Use GitLab and MITRE ATT&CK Navigator to visualize adversary techniques\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Chris Moberly\"}],\n        \"datePublished\": \"2023-08-09\",\n      }",{"title":2195,"description":2196,"authors":2201,"heroImage":2197,"date":2203,"body":2204,"category":830,"tags":2205},[2202],"Chris Moberly","2023-08-09","If you use [MITRE ATT&CK](https://attack.mitre.org/) for classifying\ncybersecurity incidents, you may want to visualize your coverage across a\nmatrix. This blog will show you how to do this automatically with GitLab by\ndeploying the [ATT&CK\nNavigator](https://github.com/mitre-attack/attack-navigator) web application\npre-populated with your own annotated matrices.\n\n\nWe make this easy by providing a fully working [example\nproject](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/gitlab-hosted-attack-navigator)\nfor you to fork and customize. When you're finished, you'll have an\ninteractive visualization tool that displays your coverage of techniques\nacross the ATT&CK framework.\n\n\n![Image showing ATT&CK Navigator deployed from example\nproject](https://about.gitlab.com/images/blogimages/2023-08-15-gitlab-mitre-attack-navigator/navigator-portal.png)\n\nATT&CK Navigator deployed from our example project\n\n{: .note.text-center}\n\n\n## About MITRE ATT&CK framework\n\nMITRE ATT&CK is a framework to classify and describe cybersecurity attacks\nbased on real-world observations. It provides a common language that can be\nused by different groups inside a security organization to collaborate on\nsecurity initiatives.\n\n\nFor example, when a company's Red Team emulates an attack based on the\ntechniques of a relevant adversary, they deliver a report that includes a\nlist of the specific technique IDs involved in the exercise. The team in\ncharge of detecting and responding to these attacks can use those IDs to\nresearch and implement improved defensive capabilities.\n\n\nBoth of these groups may want to track their coverage of offensive and\ndefensive capabilities across one of [MITRE's ATT&CK\nmatrices](https://attack.mitre.org/matrices/enterprise/). These matrices are\ncharts that visualize attack tactics and techniques relevant to specific\nindustries and technologies. For example, a company like GitLab may be\ninterested in understanding which techniques in the [Cloud\nmatrix](https://attack.mitre.org/matrices/enterprise/cloud/) we have\nemulated to test our detection and response capabilities.\n\n\nMITRE provides a free interactive web application, ATT&CK Navigator, to\nvisualize, annotate, and explore these matrices.\n\n\nAt GitLab, [our Red\nTeam](https://about.gitlab.com/handbook/security/threat-management/red-team/)\nproduces a new Navigator matrix at the completion of each operation. The\nmatrix highlights which attack techniques we've conducted. We find it useful\nto view all of these matrices in a single location, with the addition of a\ncombined matrix showing all of the techniques we have conducted across all\nhistorical operations. This helps us understand trends and identify\npotential gaps to cover in future operations.\n\n\n## Fork our example project to get started\n\nYou can fork our example project to get going on your own. First, you will\nneed an account on [GitLab.com](https://gitlab.com), or on a self-hosted\ninstance [with GitLab Pages\nenabled](https://docs.gitlab.com/ee/administration/pages/).\n\n\nNext, browse to [our example\nproject](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/gitlab-hosted-attack-navigator)\nand click on the \"Forks\" button to create a new fork. Fill in the form shown\nbelow by choosing a name, location, and description for your copy of the\nproject. You may wish to change \"Visibility level\" to \"Private,\" which will\nrequire authentication to view your deployed application.\n\n\n![Example project showing where to click on the Forks\nbutton](https://about.gitlab.com/images/blogimages/2023-08-15-gitlab-mitre-attack-navigator/fork-project.png)\n\n\nAfter that, take a look inside the `layers/` folder of your new project.\nEach file in this folder will be used to pre-populate a new tab in the\ndeployed web application where the listed attack techniques are annotated in\ngreen. We provide two example files, as well as an empty template file\ninside `templates/template.yml`.\n\n\nTemplates should list specific MITRE ATT&CK \"technique\" IDs as bullets below\ntheir corresponding MITRE ATT&CK tactic. You only need to include the\ntactics for which you will add techniques. Here is a short example of\ntechniques across four tactics:\n\n\n```yaml\n\nname: Operation 1\n\ndescription: Example of a Red Team operation - not real data\n\ntechniques:\n  initial-access:\n    - T1190\n  privilege-escalation:\n    - T1611\n    - T1055\n  lateral-movement:\n    - T1210\n    - T1021\n  exfiltration:\n    - T1041\n```\n\n\nYou can edit the example files or delete them and add new files of your own.\n\n\nThe next step is to build and deploy the web application. If you made any\nchanges to the `layers/` folder inside your default branch, this build\nprocess should have started automatically. If you just want to deploy the\napplication using the provided sample data, browse to \"Build\" -\\>\n\"Pipelines\" using the project's sidebar and click the \"Run pipeline\" button.\n\n\n![Image showing how to build and deploy app with run pipeline\nbutton](https://about.gitlab.com/images/blogimages/2023-08-15-gitlab-mitre-attack-navigator/run-pipeline.png)\n\n\n\nYou can check to see if a pipeline has completed successfully by browsing to\n\"Build\" -\\> \"Jobs\" in the sidebar. There should be at least two jobs with a\nstatus of \"passed\" - one named `build_navigator` and one named `pages`.\n\n\nOnce both jobs are complete, browse to \"Deploy\" -\\> \"Pages\" in the project's\nsidebar. Check the \"Use unique domain\" box and click the \"Save changes\"\nbutton.\n\n\n![Image showing complete jobs with save changes\nbutton](https://about.gitlab.com/images/blogimages/2023-08-15-gitlab-mitre-attack-navigator/pages-settings.png)\n\n\n\nThat's it! Your application should now be accessible using the URL provided\nunder \"Access pages\" on the same page.\n\n\n## How the example project works\n\nThere are three main components inside the example project:\n\n- The `gitlab-ci.yml` file. This defines what jobs run, and when they run.\nThe file contains rules to automatically run the jobs whenever you push\nchanges to the `layers/` folder inside the project's default branch, or\nwhenever you manually run a pipeline as described above.\n\n- The `create-layers.py` file. This Python script takes the custom YAML\nfiles you provide, and then generates individual and combined JSON files in\nthe format that ATT&CK Navigator expects. You can customize this file to\nchange the annotation color (`HIGHLIGHT_COLOR`) as well as some of the\napplication defaults (`JSON_BLOB`). In our internal version, for example, we\nremove some of the platform filters that don't apply to us.\n\n- The `build-navigator.sh` file. The shell script is responsible for\ndownloading the ATT&CK Navigator application, injecting the custom layers we\ncreate, and building the application. The output is a static website that we\nthen host using GitLab Pages.\n\n\n## More features of the ATT&CK Navigator app\n\nOnce deployed, the application is interactive. This blog focuses on using a\ncolored annotation to visualize coverage, but the [application is capable of\nmuch\nmore](https://github.com/mitre-attack/attack-navigator/blob/master/USAGE.md).\n\n\nHere are some features we use often:\n\n- In the \"layer controls\" bar up top, click on the button that looks like an\nasterisk surrounded by up and down arrows. The name is \"expand annotated\nsub-techniques.\" This will ensure that all techniques in your YAML file are\neasily viewable, as subtechniques may be hidden away when the matrix first\nloads.\n\n- If you want to hide all unannotated techniques, click anywhere in the\nmatrix and select \"select unannotated.\" Then, under \"technique controls,\"\nclick on the \"toggle state\" button. This will give you a much cleaner\nmatrix, showing only the specific techniques you've provided in your custom\nYAML files.\n\n\n![ATT&CK Navigator with unannotated techniques\nhidden](https://about.gitlab.com/images/blogimages/2023-08-15-gitlab-mitre-attack-navigator/navigator-collapsed.png)\n\nATT&CK Navigator with unannotated techniques hidden\n\n{: .note.text-center}\n\n\nATT&CK is not a bingo card, and for most organizations the goal shouldn't be\nto have green squares across the board. Instead, you should use this to\nunderstand your own coverage and how it relates to the threats most relevant\nto you.\n\n\n## Share your feedback\n\nMITRE ATT&CK Navigator is a great tool for visualizing coverage of attack\ntechniques across a matrix. Using our example project, you can easily deploy\nan automated solution that builds this tool and pre-populates it with your\nown data.\n\n\nOn our Red Team, we love to find creative use cases for GitLab, and this is\none we use ourselves. If you find this useful, or if you have any ideas to\nimprove it, we would love to hear from you! Feel free to open an issue or a\nmerge request inside our [example\nproject](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/gitlab-hosted-attack-navigator).\n",[830,743,9,108],{"slug":2207,"featured":6,"template":700},"gitlab-mitre-attack-navigator","content:en-us:blog:gitlab-mitre-attack-navigator.yml","Gitlab Mitre Attack Navigator","en-us/blog/gitlab-mitre-attack-navigator.yml","en-us/blog/gitlab-mitre-attack-navigator",{"_path":2213,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2214,"content":2220,"config":2227,"_id":2229,"_type":13,"title":2230,"_source":15,"_file":2231,"_stem":2232,"_extension":18},"/en-us/blog/gitlab-pages-features-review-apps-and-multiple-website-deployment",{"title":2215,"description":2216,"ogTitle":2215,"ogDescription":2216,"noIndex":6,"ogImage":2217,"ogUrl":2218,"ogSiteName":685,"ogType":686,"canonicalUrls":2218,"schema":2219},"GitLab Pages features review apps and multiple website deployment","GitLab Pages helps organizations reap the rewards of knowledge management, including better collaboration and accessibility. Learn how to use a new feature, Parallel Deployments.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674550/Blog/Hero%20Images/blog-image-template-1800x945__1_.png","https://about.gitlab.com/blog/gitlab-pages-features-review-apps-and-multiple-website-deployment","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab Pages features review apps and multiple website deployment\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Matthew Macfarlane\"},{\"@type\":\"Person\",\"name\":\"Janis Altherr\"}],\n        \"datePublished\": \"2024-09-23\",\n      }",{"title":2215,"description":2216,"authors":2221,"heroImage":2217,"date":2223,"body":2224,"category":694,"tags":2225,"updatedDate":2226},[2222,1563],"Matthew Macfarlane","2024-09-23","[GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/) has long been\na popular choice for hosting static websites, allowing users to showcase\ntheir projects, blogs, and documentation directly from their repositories.\n\n\nBefore GitLab 17.4, you could only have a single version of your GitLab\nPages website. So you couldn’t preview your changes or have multiple\nversions of your website deployed simultaneously. Now, with a Premium or\nUltimate license, you can do both!\n\n\n### Introducing Parallel Deployments\n\n\nWith Parallel Deployments, users can now easily preview changes and manage\nmultiple environments for their GitLab Pages sites. This enhancement allows\nseamless experimentation with new ideas, enabling users to confidently test\nand refine their sites. By catching any issues early, users can ensure the\nlive site remains stable and polished, building on the already great\nfoundation of GitLab Pages.\n\n\n### Why Parallel Deployments is a game-changer\n\n\n1. **Version control made easy**\\\n   If your project involves software development or documentation that covers multiple versions (such as user guides for different software releases), Parallel Deployments makes it easy to manage. Or you can use the feature to localize your website for different languages.\n2. **Flexibility to experiment**\\\n   Want to try out a new design or feature? Parallel Deployments lets you experiment freely. You can create a separate version of your site to test new ideas without impacting the current site. This flexibility encourages creativity and continuous improvement.\n\n### How to add review apps to your GitLab Pages project\n\n\nTo add a review app to your GitLab Pages project, edit your `.gitlab-ci.yml`\nfile to create a deployment for each merge request (MR). Let’s assume you\nstart with a `.gitlab-ci.yml` file somewhat like this:\n\n\n```yaml\n\ncreate-pages:\n  stage: deploy\n  script:\n    - npm run build\n  pages: \n    publish: dist # the name of the folder containing the pages files\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # only run this job when there's a commit to the default branch\n```\n\n\nTo also run the pages pipeline when there’s an MR being opened or updated,\nwe can add another rule to `pages.rules`:\n\n\n```yaml\n\n- if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n\n```\n\n\nIf we only add this rule, however, each Pages job will always replace the\nmain deployment – each time an MR is opened! You likely don’t want that to\nhappen.\n\n\nTo provide each individual deployment with its own URL, we’ve introduced the\nnew `pages.path_prefix` property.\n\n\nA Pages deployment with this configuration...\n\n\n```yaml\n\ncreate-pages:\n  script:\n    - ...\n  pages:\n    ...\n    path_prefix: my-review-app\n```\n\n\n...will be available at\n`https://my-pages-app-7fe824.gitlab.io/my-review-app`, or, with unique\ndomains disabled, `https://my-group.gitlab.io/my-project/my-review-app`.\n\n\nBut there’s no need to hardcode the path_prefix. You can dynamically\ngenerate it using CI variables. That’s particularly useful for review apps –\nto create a path for each MR, use the `CI_MERGE_REQUEST_IID variable`:\n\n\n```yaml\n\ncreate-pages:\n  script:\n    - ...\n  pages:\n    ...\n    path_prefix: mr-$CI_MERGE_REQUEST_IID\n```\n\n\nAn MR with the ID 114 would then automatically create a deployment at\n`https://my-pages-app-7fe824.gitlab.io/mr-114`.\n\n\nWith those concepts at hand, we’d like our pipeline to dynamically create\neither a main deployment for the default branch, or a path_prefixed-review\napp for MR events.\n\n\nFirst, let’s add a `create-pages-review-app` job to our pipeline config:\n\n\n```yaml\n\ncreate-pages-deployment:\n  # This job will create a pages deployment without path_prefix\n  # when there is a commit to the default branch\n  stage: deploy\n  script:\n    - npm run build\n  pages: \n    publish: dist \n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n\ncreate-pages-review-app:\n  # This job will create a pages deployment with a path_prefix\n  # when there a merge request is created or updated.\n  stage: deploy\n  script:\n    - npm run build\n  pages:\n    publish: dist \n    path_prefix: 'mr-$CI_MERGE_REQUEST_IID' # Prefix with the mr-\u003Ciid>, like `mr-123`\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n```\n\n\nNow you’re creating a deployment both when pushing to the default branch,\nand prefixed parallel deployments when creating or updating MRs!\n\n\nFor the best experience, add the URL to the environment job property. This\nwill add a link to the review app to the MR page:\n\n\n```yaml\n\ncreate-pages-deployment:\n  # This job will create a pages deployment without path_prefix\n  # when there is a commit to the default branch\n  stage: deploy\n  script:\n    - npm run build\n  pages: \n    publish: dist \n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n\ncreate-pages-review-app:\n  # This job will create a pages deployment with a path_prefix\n  # when there a merge request is created or updated.\n  stage: deploy\n  script:\n    - npm run build\n  pages:\n    publish: dist \n    path_prefix: 'mr-$CI_MERGE_REQUEST_IID' # Prefix with the mr-\u003Ciid>, like `mr-123`\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n  environment:\n    name: \"Pages Review MR ${CI_MERGE_REQUEST_IID}\"\n    url: $CI_PAGES_URL\n```\n\n\nCongratulations, you’ve now set up MR review apps for your Pages site.\n\n\n## How to deploy documentation for different versions of your product\n\n\nThe Parallel Deployments feature is also a useful tool if you maintain the\ndocumentation of multiple versions of your software simultaneously.\n\n\nThe below CI config will not only create a pages deployment when there is a\ncommit to the default branch, but also for any commit to branches named\n`v1`, `v2`, or `v3`.\n\n\n```yaml\n\ncreate-pages:\n  stage: deploy\n  script:\n    - ...\n  variables:\n    PAGES_PREFIX: \"$CI_COMMIT_BRANCH\" # Use the branch name by default\n  pages:\n    path_prefix: \"$PAGES_PREFIX\" # use whatever value is set in the variable\n  environment:\n    name: \"Pages ${PAGES_PREFIX}\"\n    url: $CI_PAGES_URL\n  artifacts:\n    paths:\n    - public\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n      variables:\n        PAGES_PREFIX: '' # No prefix\n    - if: $CI_COMMIT_BRANCH == 'v1'\n    - if: $CI_COMMIT_BRANCH == 'v2'\n    - if: $CI_COMMIT_BRANCH == 'v3'\n```\n\n\nBy using the `$CI_COMMIT_BRANCH` variable as the path_prefix value, each of\nthese branches will deploy their documentation to their own sub-path of your\nwebsite:\n\n\n- The branch named v1 has its docs published to \u003Cmy-domain>/v1.\n\n- The branch named v2 has its docs published to \u003Cmy-domain>/v2.\n\n- The branch named v3 has its docs published to \u003Cmy-domain>/v3.\n\n\nA new commit to one of these branches will then trigger a new deployment to\nits respective path, keeping the documentation of multiple versions up to\ndate.\n\n\nThe Parallel Deployments feature is a significant upgrade to GitLab Pages,\noffering a more flexible and efficient way to manage your knowledge. Whether\nyou're working on a small project or a large-scale site with multiple\nversions, this new capability will make your workflow smoother and more\nefficient\n\n\n> Visit our [Parallel Deployments\ndocumentation](https://docs.gitlab.com/ee/user/project/pages/#create-multiple-deployments)\nto get started today!\n\n\n### Feedback\n\n\nShare your ideas and other comments in our [feedback\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/482040)!\n",[996,108,851,696,694,9],"2025-04-09",{"slug":2228,"featured":6,"template":700},"gitlab-pages-features-review-apps-and-multiple-website-deployment","content:en-us:blog:gitlab-pages-features-review-apps-and-multiple-website-deployment.yml","Gitlab Pages Features Review Apps And Multiple Website Deployment","en-us/blog/gitlab-pages-features-review-apps-and-multiple-website-deployment.yml","en-us/blog/gitlab-pages-features-review-apps-and-multiple-website-deployment",{"_path":2234,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2235,"content":2241,"config":2248,"_id":2250,"_type":13,"title":2251,"_source":15,"_file":2252,"_stem":2253,"_extension":18},"/en-us/blog/gitops-with-gitlab-auto-devops",{"title":2236,"description":2237,"ogTitle":2236,"ogDescription":2237,"noIndex":6,"ogImage":2238,"ogUrl":2239,"ogSiteName":685,"ogType":686,"canonicalUrls":2239,"schema":2240},"Connecting Kubernetes clusters to GitLab with Auto DevOps","This is the 6th article in a series of tutorials on how to do GitOps with GitLab","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663397/Blog/Hero%20Images/logoforblogpost.jpg","https://about.gitlab.com/blog/gitops-with-gitlab-auto-devops","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitOps with GitLab: Connecting GitLab with a Kubernetes cluster - Auto DevOps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Viktor Nagy\"}],\n        \"datePublished\": \"2022-02-08\",\n      }",{"title":2242,"description":2237,"authors":2243,"heroImage":2238,"date":2245,"body":2246,"category":785,"tags":2247},"GitOps with GitLab: Connecting GitLab with a Kubernetes cluster - Auto DevOps",[2244],"Viktor Nagy","2022-02-08","_It is possible to use GitLab as a best-in-class GitOps tool, and this blog\npost series is going to show you how. These easy-to-follow tutorials will\nfocus on different user problems, including provisioning, managing a base\ninfrastructure, and deploying various third-party or custom applications on\ntop of them. You can find the entire \"Ultimate guide to GitOps with GitLab\"\ntutorial series\n[here](/blog/the-ultimate-guide-to-gitops-with-gitlab/)._\n\n\nIn this article we will look at how one can use Auto DevOps with all its\nbells and whistles to easily manage deployments.\n\n\n## Prerequisites\n\n\nThis article builds upon the previous tutorials in this series. We will\nassume that you have a Kubernetes cluster connected to GitLab using the\nGitLab Agent for Kubernetes, and you understand how the CI/CD tunnel works.\n\n\nIf this is not the case, I recommend to follow the previous articles to have\na similar setup from where we will start today.\n\n\n## What is Auto DevOps\n\n\nAuto DevOps is GitLab's answer to the complexity of software application\ndelivery. It is a set of opinionated templates that can be used \"as-is\" or\ncan be used to fast-track your own pipeline building. For some setups it\nworks from testing through various security and compliance checks to canary\ndeployments. Even if you have a less supported setup, you should be able to\nreuse some of its components, from security linting to deployment.\n\n\nYou can read more about the various [features built into Auto DevOps in our\ndocumentation](https://docs.gitlab.com/ee/topics/autodevops/).\n\n\n## The plan for building and deploying a minimul application\n\n\nThe plan for this article is to build and deploy a minimal application. The\nfocus will be on showing how you can get started quickly, without any\nmodifications on the Auto Deploy pipelines.\n\n\nThis setup will use the already known CI/CD tunnel. There will be a separate\narticle that shows how to replace the \"Auto Deploy\" part of Auto DevOps with\nGitOps style deployments.\n\n\nIn this article, we will deploy a simple hello world application. This is\nnot a tutorial about Auto DevOps, so we will only focus on the setup needed\nwhen used together with the GitLab Agent for Kubernetes.\n\n\nYou can see the final repository under\nhttps://gitlab.com/gitlab-examples/ops/gitops-demo/hello-world-service/.\n\n\n## How to build the application\n\n\nIn this section we will create our super simple hello world application and\nput a Dockerfile beside it.\n\n\n1. Start a new project.\n\n1. Add `src/main.py` with the following content:\n    ```python\n    # From https://gist.github.com/davidbgk/b10113c3779b8388e96e6d0c44e03a74\n    import http.server\n    import socketserver\n    from http import HTTPStatus\n\n    class Handler(http.server.SimpleHTTPRequestHandler):\n        def do_GET(self):\n            self.send_response(HTTPStatus.OK)\n            self.end_headers()\n            self.wfile.write(b'Hello world')\n\n    httpd = socketserver.TCPServer(('', 5000), Handler)\n    httpd.serve_forever()\n    ```\n1. Create the `Dockerfile` with:\n   ```\n   FROM python:3.9.10-slim-bullseye\n\n   WORKDIR /app\n\n   COPY ./src .\n\n   EXPOSE 5000\n\n   CMD [ \"python\", \"main.py\" ]\n   ```\n1. Commit the change to the repository.\n\n\n## How to set up Auto DevOps\n\n\n1. [Share the CI/CD\ntunnel](https://docs.gitlab.com/ee/user/clusters/agent/work_with_agent.html)\nwith the hello-world project. Note, that the Agent configuration project amd\nthe application project should be in the same project hierarchy and the\nAgent configuration project needs to be higher in this hierarchy.\n    ```yaml\n    ci_access:\n      # This agent is accessible from CI jobs in projects in these groups\n      projects:\n        - id: \u003Cpath>/\u003Cto>/\u003Cyour>/\u003Cproject>\n    ```\n1. Find out the Kubernetes context name. The agent context name is\n`\u003Cnamespace>/\u003Cgroup>/\u003Cproject>:\u003Cagent-name>`. You can see the available\ncontexts in CI with the following job:\n    ```yaml\n    contexts:\n      stage: .pre\n      image:\n        name: bitnami/kubectl:latest\n        entrypoint: [\"\"]\n      script:\n        - kubectl config get-contexts \n    ```\n1. Create your `.gitlab-ci.yml` file to have Auto DevOps working:\n    ```yaml\n    include:\n        template: Auto-DevOps.gitlab-ci.yml\n\n    variables:\n        # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.\n        KUBE_INGRESS_BASE_DOMAIN: 74.220.23.215.nip.io\n        KUBE_CONTEXT: \"gitlab-examples/ops/gitops-demo/k8s-agents:demo-agent\"\n        KUBE_NAMESPACE: \"demo-agent\"\n\n        # Feel free to enable any of these\n        TEST_DISABLED: \"true\"\n        CODE_QUALITY_DISABLED: \"true\"\n        LICENSE_MANAGEMENT_DISABLED: \"true\"\n        BROWSER_PERFORMANCE_DISABLED: \"true\"\n        LOAD_PERFORMANCE_DISABLED: \"true\"\n        SAST_DISABLED: \"true\"\n        SECRET_DETECTION_DISABLED: \"true\"\n        DEPENDENCY_SCANNING_DISABLED: \"true\"\n        CONTAINER_SCANNING_DISABLED: \"true\"\n        DAST_DISABLED: \"true\"\n        REVIEW_DISABLED: \"true\"\n        CODE_INTELLIGENCE_DISABLED: \"true\"\n        CLUSTER_IMAGE_SCANNING_DISABLED: \"true\"\n        POSTGRES_ENABLED: \"false\"\n    ```\n1. Commit the changes.\n\n\nAs you can see, I disabled many Auto DevOps functionalities in the above CI\nYAML. I did this for two reasons:\n\n\n1. Some of these features require a Premium or Ultimate license or tests in\nthe repo. I wanted to keep this tutorial \"stable\" for everyone.\n\n1. Every use case differs a little bit and Auto DevOps allows a large number\nof customizations. I wanted to highlight this by showing you the most basic\nones. Read more about [customizing Auto\nDevOps](https://docs.gitlab.com/ee/topics/autodevops/customize.html). If you\nwould like [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/)\nsupport, just remove the `REVIEW_DISABLED` line.\n\n\nThere are actually only three settings to get the Auto DevOps pipeline up\nand running:\n\n\n- The `KUBE_CONTEXT` specifies the context used for the connection, it's\nprovided by the GitLab Agent for Kubernetes.\n\n- The `KUBE_NAMESPACE` specifies the Kubernetes namespace to target with the\ndeployments. This namespace will be used as we apply the Helm charts used\nbehind the hood.\n\n- The `KUBE_INGRESS_BASE_DOMAIN` sets up an Ingress and enables user\nfriendly access to the deployed service. \n\n\n## Recap\n\n\nA very common setup I see with GitLab customers is that the development team\nis responsible for writing the application code and packaging it into a\nDocker container. During this process, they take care of basic testing as\nwell, but they are not familiar with all the security and compliance\nrequirements or the deployment pipelines used within the company. The\npresented setup and the Auto DevOps suite of templates serves these teams.\nAs you can see, the teams need minimal GitLab CI setup to run a complex\npipeline that can take care of many of their requirements.\n\n\n## What's next\n\n\nIn the next article, I will show you how to deploy an application project\nwith a GitOps style workflow.\n\n\n_[Click here](/blog/the-ultimate-guide-to-gitops-with-gitlab/)\nfor the next tutorial._\n",[549,1241,9],{"slug":2249,"featured":6,"template":700},"gitops-with-gitlab-auto-devops","content:en-us:blog:gitops-with-gitlab-auto-devops.yml","Gitops With Gitlab Auto Devops","en-us/blog/gitops-with-gitlab-auto-devops.yml","en-us/blog/gitops-with-gitlab-auto-devops",{"_path":2255,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2256,"content":2262,"config":2268,"_id":2270,"_type":13,"title":2271,"_source":15,"_file":2272,"_stem":2273,"_extension":18},"/en-us/blog/gitops-with-gitlab-manage-the-agent",{"title":2257,"description":2258,"ogTitle":2257,"ogDescription":2258,"noIndex":6,"ogImage":2259,"ogUrl":2260,"ogSiteName":685,"ogType":686,"canonicalUrls":2260,"schema":2261},"Self-managing Kubernetes agent installation with GitOps","This is the eighth and last article in a series of tutorials on how to do GitOps with GitLab.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749670178/Blog/Hero%20Images/GitLab-Ops.png","https://about.gitlab.com/blog/gitops-with-gitlab-manage-the-agent","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitOps with GitLab: Turn a GitLab agent for Kubernetes installation to manage itself\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Viktor Nagy\"}],\n        \"datePublished\": \"2022-03-30\",\n      }",{"title":2263,"description":2258,"authors":2264,"heroImage":2259,"date":2265,"body":2266,"category":785,"tags":2267},"GitOps with GitLab: Turn a GitLab agent for Kubernetes installation to manage itself",[2244],"2022-03-30","\n\n_It is possible to use GitLab as a best-in-class GitOps tool, and this blog post series is going to show you how. These easy-to-follow tutorials will focus on different user problems, including provisioning, managing a base infrastructure, and deploying various third-party or custom applications on top of them. You can find the entire \"Ultimate guide to GitOps with GitLab\" tutorial series [here](/blog/the-ultimate-guide-to-gitops-with-gitlab/)._\n\nIn this article, we will build upon the first few articles, and will turn a GitLab agent for Kubernetes installation to manage itself. This is highly recommended for production usage as it puts your `agentk` deployment under your GitOps project, and enables flawless and simple upgrades.\n\n## Prerequisites\n\nThis article builds on a few previous articles from this series and makes the following assumptions:\n\n- You have [an agent connection set up using the `kpt` based method](/blog/gitops-with-gitlab-connecting-the-cluster/).\n- You have [set up Bitnami's Sealed secrets](/blog/gitops-with-gitlab-secrets-management/).\n- You understand [how to use `kustomize` with the agent](/blog/gitops-with-gitlab/).\n\n## The goal\n\nThe goal of this tutorial is to manage a GitLab agent for Kubernetes deployment using that given agent. This has several benefits, including: \n\n- By turning the agent to manage itself, the agent configuration and deployment is managed in code. As a result, all the code-oriented tools, including Merge Requests, Approvals, and branching are there to support your processes and policies.\n- Managing a fleet of agent installations in code enables simple upgrades of the deployments.\n\n### Upgrading GitLab and the GitLab agent for Kubernetes\n\nA single GitLab instance might have dozens of agent connections. How should you upgrade all these deployments in a coordinated way? Turning everything into code simplifies the upgrade process a lot.\n\nWe have the GitLab - Agent [version compatibility documented](https://docs.gitlab.com/ee/user/clusters/agent/install/index.html#upgrades-and-version-compatibility). The recommended approach is to first upgrade GitLab together with `KAS`, the GitLab-side component of the connection, and then upgrade all the `agentk` deployments. \n\nIf you manage the `agentk` deployments in code, the upgrade requires only bumping the version number in code and the `agentk` instances will take care of upgrading themselves.\n\n## Turning an agent installation to manage itself\n\nLet's do a quick recap and an overview how we wil use the tools.\n\nWe use `kpt` to check out tagged `agentk` deployment manifests. As the manifests are a set of `kustomize` layers, we can extend them with our own overlays if needed, or just customize the setup per our requirements. The agent connection requires a token to authenticate with GitLab. We can use Bitnami's Sealed Secrets to store an encrypted sycret in the repo.\n\nAll the above code can be put under version control safely. Moreover, we can use GitLab CI/CD to dehydrate the `kustomize` package into vanilla Kubernetes manifests that the agent can deal with.\n\nLet's see the above in action!\n\n### Kustomize layer with encrypted secret\n\nBased on the previous articles, we have the `kpt` package checked out under `packages/gitlab-agent`. We would like to store the vanilla Kubernetes manifests in the repository. We can run `kustomize build packages/gitlab-agent/cluster > kubernetes/gitlab-agent.yaml` to get the manifests, but this will include the unencrypted authentication token too.\n\nTo never output the unencrypted token, we should turn it into a sealed secret.\n\nNavigate to the `gitlab-agent` Terraform project, and create a Kubernetes secret from the token `terraform output -raw token_secret | kubectl create secret generic gitlab-agent-token -n gitlab-agent --dry-run=client --type=Opaque --from-file=token=/dev/stdin -o yaml > ../../ignored/gitlab-agent-token.yaml`. If you followed the instructions in the previous articles, the files under the `ignored` directory are never committed to `git`.\n\nWe will turn this unencrypted secret into a sealed secret. As the secret will already exist in the cluster, we should instruct the Bitnami Sealed Secret controller to pull it under its management. Moreover, as kustomize applies a random hash to every secret name, we should enable renaming the secret within the namespace. We can achieve these by adding two annotations to the unencrypted secrets object.\n\nAdd the following annotations to `ignored/gitlab-agent-token.yaml`\n\n```\nannotations:\n  sealedsecrets.bitnami.com/managed: \"true\"\n  sealedsecrets.bitnami.com/namespace-wide: \"true\"\n```\n\nNext, we should create an encrypred secret from the ignored, unencrypted one running `bin/seal-secret ignored/gitlab-agent-token.yaml > packages/gitlab-agent/sealed-secret` in the root of our project. This creates the encrypted secret under `packages/gitlab-agent/sealed-secret/SealedSecret.gitlab-agent-token.yaml`. Now, we need a kustomize layer that will use this secret instead of the original one that came with `kpt`. Let's create the following files around the encrypted secret:\n\n- Create `packages/gitlab-agent/sealed-secret/kustomization.yaml` as:\n\n```yaml\napiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n- ../base\n- SealedSecret.gitlab-agent-token.yaml\ncomponents:\n- ../cluster/components/gitops-read-all\n- ../cluster/components/gitops-write-all\n- ../cluster/components/cilium-alert-read\nconfigurations:\n- configuration/sealed-secret-config.yaml\nsecretGenerator:\n- name: gitlab-agent-token\n  behavior: replace\n  type: Opaque\n  namespace: gitlab-agent\n  options:\n    annotations:\n      sealedsecrets.bitnami.com/managed: \"true\"\n      sealedsecrets.bitnami.com/namespace-wide: \"true\"\n```\n\n- Create `packages/gitlab-agent/sealed-secret/configuration/sealed-secret-config.yaml` as:\n\n```yaml\nnameReference:\n- kind: Secret\n  fieldSpecs:\n  - kind: SealedSecret\n    path: metadata/name\n  - kind: SealedSecret\n    path: spec/template/metadata/name\n```\n\nThis configuration enables us to reference the name of the Sealed Secret in the `secretGenerator`.\n\nWe created a new `kustomize` overlay that builds on the `base` and `cluster` layers, but will use the sealed secret. We can hydrate this into vanilla manifests using `kustomize build packages/gitlab-agent/sealed-secret > kubernetes/gitlab-agent.yaml`. This configuration does not include any unencrypted, sensitive data. As a result, we can commit it freely using `git commit`.\n\n### Adopt the agent by the agent\n\nRight now the agent configuration file looks similar to: \n\n```yaml\ngitops:\n  # Manifest projects are watched by the agent. Whenever a project changes,\n  # GitLab deploys the changes using the agent.\n  manifest_projects:\n  - id: path/to/your/project\n    default_namespace: gitlab-agent\n    # Paths inside of the repository to scan for manifest files.\n    # Directories with names starting with a dot are ignored.\n    paths:\n    - glob: 'kubernetes/test_config.yaml'\n    - glob: 'kubernetes/**/*.yaml'\n```\n\nIf we would push the previously hydrated manifests, `agentk` would fail applying them complaining about missing inventories. We can easily fix this by temporarily setting a looser inventory policy:\n\n```yaml\ngitops:\n  # Manifest projects are watched by the agent. Whenever a project changes,\n  # GitLab deploys the changes using the agent.\n  manifest_projects:\n  - id: path/to/your/project\n    default_namespace: gitlab-agent\n    inventory_policy: adopt_all\n    # Paths inside of the repository to scan for manifest files.\n    # Directories with names starting with a dot are ignored.\n    paths:\n    - glob: 'kubernetes/test_config.yaml'\n    - glob: 'kubernetes/**/*.yaml'\n```\n\nWith the inventory policy configured, we can commit and push our changes to GitLab. The agent will see the new configuration and resources, and will apply them into the cluster. From now on, you can change the code in the repository, push it to git, and the changes will be automatically applied into your cluster.\n\n#### What are inventory policies?\n\nThe GitLab agent for Kubernetes knows about the managed resources using so-called inventory objects. In technical terms, an inventory object is just a `ConfigMap` with a unique label. Whenever the agent sees an object that it should manage, it applies the same label. This way, every agent can easily find the resources that it manages.\n\nYou can read more about the possible [inventory policy configurations in the documentation](https://docs.gitlab.com/ee/user/infrastructure/clusters/deploy/inventory_object.html).\n\n\n#### A word about RBAC\n\nDepending on the authorization rights given to the `agentk` deployment, not every change might be possible. For example, if you would like to create new `ClusterRole` and `ClusterRoleBinding` in a new `kustomize` overlay, and apply that with the Agent, that might fail. It will fail, if your current role-based access control (RBAC) does not allow your `agentk` deployment to create these resources. In this case, you should either provide higher rights to your `agentk` service account first or you should apply the changes manually from your command line.\n\n### Automatic hydration\n\nNow, if you want to change something in your agent deployment, you need to take two actions:\n\n- change the code in the `kpt` package\n- run `kustomize build` to hydrate the results\n\nLet's automate the second step so you can focus on your main job only. Following the setup of [a GitOps-style Auto DevOps pipeline](/blog/gitops-with-gitlab/#hydrating-the-manifests), we need to extend the `hydrate-packages` job:\n\n\n```yaml\nhydrate-packages:\n      ...\n      script:\n      - mkdir -p new_manifests\n      ...\n      - kustomize build packages/gitlab-agent/sealed-secret > new_manifests/gitlab-agent.yaml\n```\n\nWe can re-use all the other automation as presented in the previous articles.\n\n## How to upgrade `agentk`?\n\nJust to provide a practical example, let's see how we can use the above setup to easily upgrade an `agentk` deployment to a newer version.\n\nBy running `kustomize cfg set packages/gitlab-agent agent-version v14.9.1` we set the intended `agentk` version to be version `14.9.1`. You can commit and push this change to git, and lay back in your chair to see how the changes are being rolled out across your clusters. You can point several agent configurations at the same `kubernetes/gitlab-agent.yaml` manifest, and upgrade all of them at once.\n\n## Recap\n\nIn this article we have seen:\n\n- how to turn an Agent deployment to manage itself\n- how to extend the default `kpt` project with a custom `kustomize` overlay to customize the `agentk` deployment\n- how to easily upgrade a set of `agentk` deployments\n- how to pull already existing objects to be managed by the Agent using inventory policies\n\n_Note: This is the final installment in this series of [how to do GitOps with GitLab](/blog/the-ultimate-guide-to-gitops-with-gitlab)._\n\n\n",[549,1241,9],{"slug":2269,"featured":6,"template":700},"gitops-with-gitlab-manage-the-agent","content:en-us:blog:gitops-with-gitlab-manage-the-agent.yml","Gitops With Gitlab Manage The Agent","en-us/blog/gitops-with-gitlab-manage-the-agent.yml","en-us/blog/gitops-with-gitlab-manage-the-agent",{"_path":2275,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2276,"content":2282,"config":2289,"_id":2291,"_type":13,"title":2292,"_source":15,"_file":2293,"_stem":2294,"_extension":18},"/en-us/blog/go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss",{"title":2277,"description":2278,"ogTitle":2277,"ogDescription":2278,"noIndex":6,"ogImage":2279,"ogUrl":2280,"ogSiteName":685,"ogType":686,"canonicalUrls":2280,"schema":2281},"Go tools and GitLab: How to do continuous integration like a boss","How the team at Pantomath makes their lives easier with GitLab CI.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667978/Blog/Hero%20Images/go-tools-and-gitlab.jpg","https://about.gitlab.com/blog/go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Go tools and GitLab: How to do continuous integration like a boss\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Julien Andrieux\"}],\n        \"datePublished\": \"2017-11-27\",\n      }",{"title":2277,"description":2278,"authors":2283,"heroImage":2279,"date":2285,"body":2286,"category":785,"tags":2287},[2284],"Julien Andrieux","2017-11-27","At [Pantomath](https://pantomath.io/), we use [GitLab](/) for all our\ndevelopment work. The purpose of this paper is not to present GitLab and all\n[its features](/pricing/feature-comparison/), but to introduce how we use these tools to ease\nour lives. So what is it all about? To automate everything that is related\nto your development project, and let you focus on your code.\n\n\n\u003C!-- more -->\n\n\nWe’ll cover the [lint](https://en.wikipedia.org/wiki/Lint_(software)), [unit\ntests](https://en.wikipedia.org/wiki/Unit_testing), [data\nrace](https://en.wikipedia.org/wiki/Race_condition), [memory\nsanitizer](https://clang.llvm.org/docs/MemorySanitizer.html), [code\ncoverage](https://en.wikipedia.org/wiki/Code_coverage), and build.\n\n\nAll the source code shown in this post is available at\n[gitlab.com/pantomath-io/demo-tools](https://gitlab.com/pantomath-io/demo-tools).\nSo feel free to get the repository, and use the tags to navigate in it. The\nrepository should be placed in the `src` folder of your `$GOPATH`:\n\n\n```bash\n\n$ go get -v -d gitlab.com/pantomath-io/demo-tools\n\n$ cd $GOPATH/src/gitlab.com/pantomath-io/demo-tools\n\n```\n\n\n### Go tools\n\n\nLuckily, `Go` — the open source programming language also known as golang —\ncomes with a [lot of useful tools](https://golang.org/cmd/go/), to build,\ntest, and check your code. In fact, it’s all there. We’ll just add extra\ntools to glue them together. But before we go there, we need to take them\none by one, and see what they do.\n\n\n#### Package list\n\n\nYour Go project is a collection of packages, as described in the [official\ndoc](https://golang.org/doc/code.html). Most of the following tools will be\nfed with these packages, and thus the first command we need is a way to list\nthe packages. Hopefully, the `Go` language covers our back with the `list`\nsubcommand ([read the fine\nmanual](https://golang.org/cmd/go/#hdr-List_packages) and this [excellent\npost from Dave\nCheney](https://dave.cheney.net/2014/09/14/go-list-your-swiss-army-knife)):\n\n\n```bash\n\n$ go list ./...\n\n```\n\n\nNote that we want to avoid applying our tools on external packages or\nresources, and restrict it to **our** code. So we need to get rid of the\n[vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories):\n\n\n```bash\n\n$ go list ./... | grep -v /vendor/\n\n```\n\n\n#### Lint\n\n\nThis is the very first tool we use on the code: the linter. Its role is to\nmake sure that the code respects the code style. This may sounds like an\noptional tool, or at least a “nice-to-have” but it really helps to keep\nconsistent style over your project.\n\n\nThis linter is not part of Go *per se*, so you need to grab it and install\nit by hand (see [official doc](https://github.com/golang/lint)).\n\n\nThe usage is fairly simple: you just run it on the packages of your code\n(you can also point the `.go` files):\n\n\n```bash\n\n$ golint -set_exit_status $(go list ./... | grep -v /vendor/)\n\n```\n\n\nNote the `-set_exit_status` option. By default, `golint` only prints the\nstyle issues, and returns (with a 0 return code), so the CI never considers\nsomething went wrong. If you specify the `-set_exit_status`, the return code\nfrom `golint` will be different from 0 if any style issue is encountered.\n\n\n#### Unit test\n\n\nThese are the most common tests you can run on your code. For each `.go`\nfile, we need to have an associated `_test.go` file holding the unit tests.\nYou can run the tests for all the packages with the following command:\n\n\n```bash\n\n$ go test -short $(go list ./... | grep -v /vendor/)\n\n```\n\n\n#### Data race\n\n\nThis is usually a hard subject to cover, but the `Go` tool has it by default\n(but only available on `linux/amd64`, `freebsd/amd64`, `darwin/amd64` and\n`windows/amd64`). For more information about data race, see [this\narticle](https://golang.org/doc/articles/race_detector.html). Meanwhile,\nhere is how to run it:\n\n\n```bash\n\n$ go test -race -short $(go list ./... | grep -v /vendor/)\n\n```\n\n\n#### Memory sanitizer\n\n\nClang has a nice detector for uninitialized reads called\n[MemorySanitizer](https://clang.llvm.org/docs/MemorySanitizer.html). The `go\ntest` tool is kind enough to interact with this Clang module (as soon as you\nare on `linux/amd64` host and using a recent version of Clang/LLVM\n(`>=3.8.0`). This command is how to run it:\n\n\n```bash\n\n$ go test -msan -short $(go list ./... | grep -v /vendor/)\n\n```\n\n\n#### Code coverage\n\n\nThis is also a must have to evaluate the health of your code, and see what\nthe part of code is under unit tests and what part is not. [Rob\nPike](https://twitter.com/rob_pike) wrote a [full post on that very\nsubject](https://blog.golang.org/cover).\n\n\nTo calculate the code coverage ratio, we need to run the following script:\n\n\n```bash\n\n$ PKG_LIST=$(go list ./... | grep -v /vendor/)\n\n$ for package in ${PKG_LIST}; do\n    go test -covermode=count -coverprofile \"cover/${package##*/}.cov\" \"$package\" ;\ndone\n\n$ tail -q -n +2 cover/*.cov >> cover/coverage.cov\n\n$ go tool cover -func=cover/coverage.cov\n\n```\n\n\nIf we want to get the coverage report in HTML format, we need to add the\nfollowing command:\n\n\n```bash\n\n$ go tool cover -html=cover/coverage.cov -o coverage.html\n\n```\n\n\n#### Build\n\n\nLast but not least, once the code has been fully tested, we might want to\ncompile it to make sure we can build a working binary.\n\n\n```bash\n\n$ go build -i -v gitlab.com/pantomath-io/demo-tools\n\n```\n\n\n### Makefile\n\n\n*git tag:*\n[init-makefile](https://gitlab.com/pantomath-io/demo-tools/tags/init-makefile)\n\n\n![](https://cdn-images-1.medium.com/max/1600/1*Ip_q_6I-kNpUjuPMOutuTA.jpeg)\n\n*\u003Csmall>Photo by [Matt\nArtz](https://unsplash.com/photos/qJE5Svhs2ek?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\u003C/small>*\n\n\nNow we have all the tools that we may use in the context of continuous\nintegration, we can wrap them all in a\n[Makefile](https://gitlab.com/pantomath-io/demo-tools/blob/init-makefile/Makefile),\nand have a consistent way to call them.\n\n\nThe purpose of this doc is not to present `make`, but you can refer to\n[official documentation](https://www.gnu.org/software/make/manual/make.html)\nto learn more about it.\n\n    PROJECT_NAME := \"demo-tools\"\n    PKG := \"gitlab.com/pantomath-io/$(PROJECT_NAME)\"\n    PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/)\n    GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go)\n\n    .PHONY: all dep build clean test coverage coverhtml lint\n\n    all: build\n\n    lint: ## Lint the files\n      @golint -set_exit_status ${PKG_LIST}\n\n    test: ## Run unittests\n      @go test -short ${PKG_LIST}\n\n    race: dep ## Run data race detector\n      @go test -race -short ${PKG_LIST}\n\n    msan: dep ## Run memory sanitizer\n      @go test -msan -short ${PKG_LIST}\n\n    coverage: ## Generate global code coverage report\n      ./tools/coverage.sh;\n\n    coverhtml: ## Generate global code coverage report in HTML\n      ./tools/coverage.sh html;\n\n    dep: ## Get the dependencies\n      @go get -v -d ./...\n\n    build: dep ## Build the binary file\n      @go build -i -v $(PKG)\n\n    clean: ## Remove previous build\n      @rm -f $(PROJECT_NAME)\n\n    help: ## Display this help screen\n      @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = \":.*?## \"}; {printf \"\\033[36m%-30s\\033[0m %s\\n\", $$1, $$2}'\n\nWhat do we have now? One target for any tool previously presented, and three\nmore targets for:\n\n\n* installation of dependencies (`dep`);\n\n* housekeeping of the project (`clean`);\n\n* some nice and shiny help (`help`).\n\n\nNote that we also had to create a script for the code coverage work. This is\nbecause implementing loops over files in a Makefile is a pain. So the work\nis done in a `bash` script, and the Makefile only triggers this script.\n\n\nYou can try the Makefile with the following commands:\n\n    $ make help\n    $ make lint\n    $ make coverage\n\n### Continuous integration\n\n\n*git tag:*\n[init-ci](https://gitlab.com/pantomath-io/demo-tools/tags/init-ci)\n\n\nNow the tools are in place, and we can run various tests on our code, we’d\nlike to automate these, on your repository. Luckily, GitLab offers [CI\npipelines](/solutions/continuous-integration/) just for this. And the setup\nfor this is pretty straightforward: all you create is a `.gitlab-ci.yml`\nfile at the root of the repository.\n\n\nThe [full documentation](https://docs.gitlab.com/ee/ci/yaml/) on this Yaml\nfile presents all the options, but you can start with this `.gitlab-ci.yml`:\n\n\n```yaml\n\nimage: golang:1.9\n\n\ncache:\n  paths:\n    - /apt-cache\n    - /go/src/github.com\n    - /go/src/golang.org\n    - /go/src/google.golang.org\n    - /go/src/gopkg.in\n\nstages:\n  - test\n  - build\n\nbefore_script:\n  - mkdir -p /go/src/gitlab.com/pantomath-io /go/src/_/builds\n  - cp -r $CI_PROJECT_DIR /go/src/gitlab.com/pantomath-io/pantomath\n  - ln -s /go/src/gitlab.com/pantomath-io /go/src/_/builds/pantomath-io\n  - make dep\n\nunit_tests:\n  stage: test\n  script:\n    - make test\n\nrace_detector:\n  stage: test\n  script:\n    - make race\n\nmemory_sanitizer:\n  stage: test\n  script:\n    - make msan\n\ncode_coverage:\n  stage: test\n  script:\n    - make coverage\n\ncode_coverage_report:\n  stage: test\n  script:\n    - make coverhtml\n  only:\n  - master\n\nlint_code:\n  stage: test\n  script:\n    - make lint\n\nbuild:\n  stage: build\n  script:\n    - make\n```\n\n\nIf you break down the file, here are some explanations on its content:\n\n\n* The first thing is to choose what Docker image will be used to run the CI.\nHead to the [Docker Hub](https://hub.docker.com/) to choose the right image\nfor your project.\n\n* Then, you specify some folders of this image [to be\ncached](https://docs.gitlab.com/ee/ci/yaml/#cache). The goal here is to\navoid downloading the same content several times. Once a job is completed,\nthe listed paths will be archived, and next job will use the same archive.\n\n* You define the different `stages` that will group your jobs. In our case,\nwe have two [stages](https://docs.gitlab.com/ee/ci/yaml/#stages) (to be\nprocessed in that order): `test` and `build`. We could have other stages,\nsuch as `deploy`.\n\n* The `before_script`\n[section](https://docs.gitlab.com/ee/ci/yaml/#before_script) defines the\ncommands to run in the Docker container right before the job is actually\ndone. In our context, the commands just copy or link the repository deployed\nin the `$GOPATH`, and install dependencies.\n\n* Then come the actual [jobs](https://docs.gitlab.com/ee/ci/jobs/), using\nthe `Makefile` targets. Note the special case for `code_coverage_report`\nwhere execution is restricted to the `master` branch (we don’t want to\nupdate the code coverage report from feature branches for instance).\n\n\nAs we commit/push the `.gitlab-ci.yml` file in the repository, the CI is\n[automatically\ntriggered](https://gitlab.com/pantomath-io/demo-tools/pipelines/13481935).\nAnd the pipeline fails. Howcome?\n\n\nThe `lint_code`\n[job](https://gitlab.com/pantomath-io/demo-tools/-/jobs/38690212) fails\nbecause it can’t find the `golint` binary:\n\n\n```bash\n\n$ make lint\n\nmake: golint: Command not found\n\nMakefile:11: recipe for target 'lint' failed\n\nmake: *** [lint] Error 127\n\n```\n\n\nSo,\n[update](https://gitlab.com/pantomath-io/demo-toolscommit/17a0206eb626504e559f56773e2d81c7b5808dbe)\nyour `Makefile` to install `golint` as part of the `dep` target.\n\n\nThe `memory_sanitizer`\n[job](https://gitlab.com/pantomath-io/demo-tools/-/jobs/38690209) fails\nbecause `gcc` complains:\n\n\n```bash\n\n$ make msan\n\n# runtime/cgo\n\ngcc: error: unrecognized argument to -fsanitize= option: 'memory'\n\nMakefile:20: recipe for target 'msan' failed\n\nmake: *** [msan] Error 2\n\n```\n\n\nBut remember we need to use Clang/LLVM `>=3.8.0` to enjoy the `-msan` option\nin `go test` command.\n\n\nWe have two options here:\n\n\n* either we set up Clang in the job (using `before_script`);\n\n* or we use a Docker image with Clang installed by default.\n\n\nThe first option is nice, but that implies to have this setup done **for\nevery single job**. This is going to be so long, we should do it once and\nfor all. So we prefer the second option, which is a good way to play with\n[GitLab\nRegistry](https://docs.gitlab.com/ee/user/packages/container_registry/index.html).\n\n\n*git tag:*\n[use-own-docker](https://gitlab.com/pantomath-io/demo-tools/tags/use-own-docker)\n\n\nWe need to create a\n[Dockerfile](https://gitlab.com/pantomath-io/demo-tools/blob/use-own-docker/Dockerfile)\nfor the container (as usual: read the [official\ndocumentation](https://docs.docker.com/engine/reference/builder) for more\noptions about it):\n\n    # Base image:\n    FROM golang:1.9\n    MAINTAINER Julien Andrieux \u003Cjulien@pantomath.io>\n\n    # Install golint\n    ENV GOPATH /go\n    ENV PATH ${GOPATH}/bin:$PATH\n    RUN go get -u github.com/golang/lint/golint\n\n    # Add apt key for LLVM repository\n    RUN wget -O -\n     | apt-key add -\n\n    # Add LLVM apt repository\n    RUN echo \"deb\n     llvm-toolchain-stretch-5.0 main\" | tee -a /etc/apt/sources.list\n\n    # Install clang from LLVM repository\n    RUN apt-get update && apt-get install -y --no-install-recommends \\\n        clang-5.0 \\\n        && apt-get clean \\\n        && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*\n\n    # Set Clang as default CC\n    ENV set_clang /etc/profile.d/set-clang-cc.sh\n    RUN echo \"export CC=clang-5.0\" | tee -a ${set_clang} && chmod a+x ${set_clang}\n\nThe container built out of this Dockerfile will be based on\n[golang:1.9](https://hub.docker.com/_/golang/) image (the one referenced in\nthe `.gitlab-ci.yml` file).\n\n\nWhile we’re at it, we install `golint` in the container, so we have it\navailable. Then we follow [official way](http://apt.llvm.org/) of installing\nClang 5.0 from LLVM repository.\n\n\nNow we have the Dockerfile in place, we need to build the container image\nand make it available for GitLab:\n\n\n```bash\n\n$ docker login registry.gitlab.com\n\n$ docker build -t registry.gitlab.com/pantomath-io/demo-tools .\n\n$ docker push registry.gitlab.com/pantomath-io/demo-tools\n\n```\n\n\nThe first command connects you to the GitLab Registry. Then you build the\ncontainer image described in the Dockerfile. And finally, you push it to the\nGitLab Registry.\n\n\nTake a look at the [Registry for your\nrepository](https://gitlab.com/pantomath-io/demo-tools/container_registry),\nyou’ll see your image, ready to be used. And to have the CI using your\nimage, you just need to update the `.gitlab-ci.yml` file:\n\n    image: golang:1.9\n\nbecomes\n\n    image: registry.gitlab.com/pantomath-io/demo-tools:latest\n\nOne last detail: you need to tell the CI to use the proper compiler (i.e.\nthe `CC` environment variable), so we add the variable initialization in the\n`.gitlab-ci.yml` file:\n\n    export CC=clang-5.0\n\nOnce the modification are done, next commit will trigger the pipeline, which\nnow works:\n\n\n[gitlab.com/pantomath-io/demo-tools/pipelines/13497136](https://gitlab.com/pantomath-io/demo-tools/pipelines/13497136)\n\n\n### Badges\n\n\n*git tag:*\n[init-badges](https://gitlab.com/pantomath-io/demo-tools/tags/init-badges)\n\n\n![](https://cdn-images-1.medium.com/max/1600/1*0pY_6oCiHZ_eLh0vfg5rDA.jpeg)\n\n\n*\u003Csmall>Photo by [Jakob\nOwens](https://unsplash.com/photos/ZBadHaTUkP0?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\u003C/small>*\n\n\nNow the tools are in place, every commit will launch a test suite, and you\nprobably want to show it, and that’s legitimate :) The best way to do so is\nto use badges, and the best place for it is the `README`\n[file](https://gitlab.com/pantomath-io/demo-tools/blob/init-badges/README.md).\n\n\nEdit it and add the four following badges:\n\n\n* Build Status: the status of the last pipeline on the `master` branch:\n\n\n```\n\n[![Build\nStatus](https://gitlab.com/pantomath-io/demo-tools/badges/master/build.svg)](https://gitlab.com/pantomath-io/demo-tools/commits/master)\n\n```\n\n\n* Coverage Report: the percentage of source code covered by tests\n\n\n```\n\n[![Coverage\nReport](https://gitlab.com/pantomath-io/demo-tools/badges/master/coverage.svg)](https://gitlab.com/pantomath-io/demo-tools/commits/master)\n\n```\n\n\n* Go Report Card:\n\n\n```\n\n[![Go Report\nCard](https://goreportcard.com/badge/gitlab.com/pantomath-io/demo-tools)](https://goreportcard.com/report/gitlab.com/pantomath-io/demo-tools)\n\n```\n\n\n* License:\n\n\n```\n\n[![License\nMIT](https://img.shields.io/badge/License-MIT-brightgreen.svg)](https://img.shields.io/badge/License-MIT-brightgreen.svg)\n\n```\n\n\nThe coverage report needs a special configuration. You need to tell GitLab\nhow to get that information, considering that there is a job in the CI that\n*displays* it when it runs.\u003Cbr> There is a\n[configuration](https://gitlab.com/help/user/project/pipelines/settings#test-coverage-parsing)\nto provide GitLab with a regexp, used in any job’ output. If the regexp\nmatches, GitLab consider the match to be the code coverage result.\n\n\nSo head to `Settings > CI/CD` in your repository, scroll down to the `Test\ncoverage parsing` setting in the `General pipelines settings` section, and\nuse the following regexp:\n\n    total:\\s+\\(statements\\)\\s+(\\d+.\\d+\\%)\n\nYou’re all set! Head to the [overview of your\nrepository](https://gitlab.com/pantomath-io/demo-tools/tree/init-badges),\nand look at your `README`:\n\n\n### Conclusion\n\n\nWhat’s next? Probably more tests in your CI. You can also look at the CD\n([Continuous\nDeployment](https://docs.gitlab.com/ee/ci/environments/index.html)) to\nautomate the deployment of your builds. The documentation can be done using\n[GoDoc](https://godoc.org/-/about). Note that you generate a coverage report\nwith the `code_coverage_report`, but don’t use it in the CI. You can make\nthe job copy the HTML file to a web server, using `scp` (see this\n[documentation](https://docs.gitlab.com/ee/ci/ssh_keys/) on how to use SSH\nkeys).\n\n\nMany thanks to [Charles Francoise](https://dev.to/loderunner) who co-wrote\nthis paper and\n[gitlab.com/pantomath-io/demo-tools](https://gitlab.com/pantomath-io/demo-tools).\n\n\n## About the Guest Author\n\n\nJulien Andrieux is currently working on Pantomath. Pantomath is a modern,\nopen source monitoring solution, built for performance, that bridges the\ngaps across all levels of your company. The wellbeing of your infrastructure\nis everyone’s business. [Keep up with the project](http://goo.gl/tcxtXq).\n\n *[Go tools & GitLab - how to do Continuous Integration like a boss](https://medium.com/pantomath/go-tools-gitlab-how-to-do-continuous-integration-like-a-boss-941a3a9ad0b6) was originally published on Medium.*\n\n*Cover photo by [Todd\nQuackenbush](https://unsplash.com/photos/IClZBVw5W5A?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)*\n\n{: .note}\n",[108,2288,9],"user stories",{"slug":2290,"featured":6,"template":700},"go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss","content:en-us:blog:go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss.yml","Go Tools And Gitlab How To Do Continuous Integration Like A Boss","en-us/blog/go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss.yml","en-us/blog/go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss",{"_path":2296,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2297,"content":2303,"config":2310,"_id":2312,"_type":13,"title":2313,"_source":15,"_file":2314,"_stem":2315,"_extension":18},"/en-us/blog/google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab",{"title":2298,"description":2299,"ogTitle":2298,"ogDescription":2299,"noIndex":6,"ogImage":2300,"ogUrl":2301,"ogSiteName":685,"ogType":686,"canonicalUrls":2301,"schema":2302},"Google Cloud integrations for secure Cloud Run deployments at GitLab","This tutorial demonstrates how to use GitLab’s Google Artifact Management integration to deploy to Google Cloud Run, a serverless runtime for containers application.\n","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099336/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945_fJKX41PJHKCfSOWw4xQxm_1750099336757.png","https://about.gitlab.com/blog/google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Google Cloud integrations for secure Cloud Run deployments at GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Regnard Raquedan\"},{\"@type\":\"Person\",\"name\":\"Matt Genelin\"}],\n        \"datePublished\": \"2025-01-15\",\n      }",{"title":2298,"description":2299,"authors":2304,"heroImage":2300,"date":2306,"body":2307,"category":694,"tags":2308},[1874,2305],"Matt Genelin","2025-01-15","*This tutorial is from a recent Arctiq, GitLab, and Google in-person\nworkshop. The goal was to explore common security challenges faced by\norganizations as they journey to the cloud.*\n\n\nThis tutorial will help you learn about the [Google Cloud integrations in\nGitLab](https://cloud.google.com/docs/gitlab). These features are meant to\nhelp accelerate and improve security of deployments to Google Cloud.\n\n\n![Google integrations\nlist](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099345/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099345112.png)\n\n\n## Prerequisites\n\n\n1. [Google Cloud\nproject](https://cloud.google.com/resource-manager/docs/creating-managing-projects)  \n\n2. Appropriate [IAM permissions](https://cloud.google.com/iam/docs/) for\nsecurity, Artifact Registry, and Cloud Run usage. For this tutorial, ensure\nyou have the \"Owner\" role with the aforementioned project.\n\n\n## Setting up Workload Identity Federation\n\n\nIn this step, we configure GitLab to connect Google Cloud's Workload\nIdentity Federation to reduce the need for service accounts and let the two\nplatforms use short-lived credentials on-demand.\n\n\n1. On the left sidebar, select **Search** or go to and find your group or\nproject. If you configure this in a group, settings apply to all projects\nwithin by default.  \n\n2. Select **Settings \\> Integrations**.  \n\n3. Select **Google Cloud IAM**.  \n\n4. Input the Project ID and Project number in the respective fields. This\ninformation can be obtained from the Google Cloud console\n[Welcome](https://console.cloud.google.com/welcome) page of your project.  \n\n5. Input the desired Pool ID and Provider ID in the respective fields. These\nare values that you provide and must be unique from other Pool and Provider\nIDs.  \n\n6. Copy the generated command and then go to the **Google Cloud console**.  \n\n7. Run **Cloud Shell** and execute the generated command from the Workload\nIdentity Federation integration page.  \n\n8. Once successful, the **Google Cloud IAM** integration will be designated\nas active in the Integrations list at the GitLab project.\n\n\n## Artifact Registry configuration\n\n\nAs an alternative to GitLab's own place to host artifacts, deploying to\nGoogle Cloud's Artifact Registry is another way to leverage their\ninfrastructure. This section will provide steps on how to use GitLab's\nnative integration with Artifact Registry. Note that Workload Identity\nFederation must already be configured prior to this.\n\n\n1. At the **Google Cloud** console, go to **Artifact Registry** via search\nor the main navigation.  \n\n2. Create a new repository by clicking the **\"+\"** icon. At the creation\npage, provide a name and keep the **Docker** format and **Standard** mode\nselected. Select **Region** and choose **us-central1**. Leave the rest at\nthe default settings and click **Create**.  \n\n3. Once the repository is created and confirmed, go back to your GitLab\nproject.  \n\n4. In your GitLab project, on the left sidebar, select **Settings >\nIntegrations**. Then select **Google Artifact Registry**.  \n\n5. Under Enable integration, select the **Active** checkbox, then complete\nthe fields:  \n   * Google Cloud project ID: The ID of the Google Cloud project where your Artifact Registry repository is located.  \n   * Repository name: The name of your Artifact Registry repository.  \n   * Repository location: The location of your Artifact Registry repository. (`us-central1` is assumed.)  \n6. In **Configure Google Cloud IAM policies**, follow the onscreen\ninstructions to set up the IAM policies in Google Cloud. These policies are\nrequired to use the Artifact Registry repository in your GitLab project.\nSelect **Save** changes.  \n\n7. To view your Google Cloud artifacts, on the left sidebar, select **Deploy\n> Google Artifact Registry**.\n\n\n## Cloud Run configuration\n\n\n1. Enable the Cloud Run API, if not done already. Go to **APIs & Services >\nEnabled APIs & Services**. From there, click **Enable APIs & Services** at\nthe top and search for **Cloud Run Admin API**. Select the search result and\nenable the API.  \n\n2. Configure the IAM policies in Google Cloud to grant permissions to allow\nthe Cloud Run CI/CD component to deploy to Cloud Run.\n\n\n```\n\nGCP_PROJECT_ID=\"\u003CPROJECT ID>\"\n\nGCP_PROJECT_NUMBER=\"\u003CPROJECT NUMBER>\"\n\nGCP_WORKLOAD_IDENTITY_POOL=\"\u003CPOOL ID>\"\n\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/run.admin'\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/iam.serviceAccountUser'\n\ngcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \\\n  --member=\"principalSet://iam.googleapis.com/projects/${GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GCP_WORKLOAD_IDENTITY_POOL}/attribute.developer_access/true\" \\\n  --role='roles/cloudbuild.builds.editor'\n```\n\n\n## Deploy to Cloud Run\n\n\nIn this section, you will use Gitlab's CI/CD components to deploy to Cloud\nRun, Google Cloud's serverless runtime for containers.\n\n\n1. Go to the GitLab project and from the list of files in the source code,\nfind `.gitlab-ci.yaml`. Click the **file name** and the single file editor\nwill show up. Click the **Edit** button and select the **Open in Web IDE**\noption.  \n\n2. In Web IDE, copy-paste the following code:\n\n\n```\n\nstages:\n    - build\n    - upload\n    - deploy\n```\n\n\nThis code snippet sets up three stages in the pipeline: build, upload, and\ndeploy.\n\n\n1. The next step is to create two CI/CD variables in the same YAML file:\n\n\n```\n\nvariables:\n    GITLAB_IMAGE: $CI_REGISTRY_IMAGE/main:$CI_COMMIT_SHORT_SHA\n    AR_IMAGE: $GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_LOCATION-docker.pkg.dev/$GOOGLE_ARTIFACT_REGISTRY_PROJECT_ID/$GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_NAME/main:$CI_COMMIT_SHORT_SHA\n```\n\n\nThe first variable, `GITLAB\\_IMAGE`, denotes the container image that the\npipeline creates by default. The second one, `AR\\_IMAGE`, denotes the\nlocation at Google Cloud's Artifact Registry where the container image will\nbe pushed to.\n\n\n2. Next, define the code that will build the container image:\n\n\n```\n\nbuild:\n    image: docker:24.0.5\n    stage: build\n    services:\n        - docker:24.0.5-dind\n    before_script:\n        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    script:\n        - docker build -t $GITLAB_IMAGE .\n        - docker push $GITLAB_IMAGE\n```\n\n\nThis code uses [pre-defined CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\nfor the Docker commands.\n\n\n3. The final step is using two CI/CD components to deploy to Google Cloud.\nThe first component integrates with Artifact Registry and the second is the\ndeployment to Cloud Run:\n\n\n```\n\ninclude:\n    - component: gitlab.com/google-gitlab-components/artifact-registry/upload-artifact-registry@main\n      inputs:\n        stage: upload\n        source: $GITLAB_IMAGE\n        target: $AR_IMAGE\n\n    - component: gitlab.com/google-gitlab-components/cloud-run/deploy-cloud-run@main\n      inputs:\n        stage: deploy\n        project_id: \"\u003CPROJECT_ID>\"\n        service: \"tanuki-racing\"\n        region: \"\u003CREGION>\"\n        image: $AR_IMAGE\n```\n\n\nReplace \u003CPROJECT_ID> with your Google Cloud Project ID. Replace with the\n[Google Cloud region](https://cloud.google.com/compute/docs/regions-zones)\nmost appropriate to your location. `us-central1` is assumed.\n\n\nCommit the changes and push to the main branch. For reference, the final\n`.gitlab-ci.yaml` should look like this, noting to replace the \u003CPROJECT ID>\nand \u003CREGION> with the appropriate values:\n\n\n```\n\nstages:\n    - build\n    - upload\n    - deploy\nvariables:\n    GITLAB_IMAGE: $CI_REGISTRY_IMAGE/main:$CI_COMMIT_SHORT_SHA\n    AR_IMAGE: $GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_LOCATION-docker.pkg.dev/$GOOGLE_ARTIFACT_REGISTRY_PROJECT_ID/$GOOGLE_ARTIFACT_REGISTRY_REPOSITORY_NAME/main:$CI_COMMIT_SHORT_SHA\n\nbuild:\n    image: docker:24.0.5\n    stage: build\n    services:\n        - docker:24.0.5-dind\n    before_script:\n        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    script:\n        - docker build -t $GITLAB_IMAGE .\n        - docker push $GITLAB_IMAGE\n\ninclude:\n    - component: gitlab.com/google-gitlab-components/artifact-registry/upload-artifact-registry@main\n      inputs:\n        stage: upload\n        source: $GITLAB_IMAGE\n        target: $AR_IMAGE\n\n    - component: gitlab.com/google-gitlab-components/cloud-run/deploy-cloud-run@main\n      inputs:\n        stage: deploy\n        project_id: \"\u003CPROJECT_ID>\"\n        service: \"tanuki-racing\"\n        region: \"\u003CREGION>\"\n        image: $AR_IMAGE\n```\n\n\n1. Go back to the main GitLab project and view the pipeline that was just\ninitiated. Take note of the stages that should be the same stages that were\ndefined in Step 2.  \n\n2. Once the pipeline is complete, go to the Google Cloud console and then\n**Cloud Run** via search or navigation. A new Cloud Run service called\n`tanuki-racing` should be created.  \n\n3. Click the **service name** and then go to the **Security** tab. Ensure\nthat the service is set to **Allow unauthenticated invocations**. This will\nmake the deployed app publicly available. The app URL posted on screen is\nnow available and should open a new browser tab when clicked.\n\n\nBy utilizing GitLab’s CI/CD pipelines to build and push a containerized\napplication to Google Artifact Registry, you can see the power of GitLab’s\nAI-powered DevSecOps Platform as a means to building secure applications.\nGitLab also deployed the containerized application to Google’s Cloud Run as\na low-cost running application on the public internet. Using GitLab to\ninstrument building an application, pushing a container and triggering a\ncloud run deployment allows DevOps engineers to have the assurance that\nsecure applications are being run on the public-facing internet.\n\n\n> [Sign up for a free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/devsecops/) to begin working\nwith these integrations. Also, check out our [solutions architecture\narea](https://about.gitlab.com/blog/tags/solutions-architecture/) for more\nGitlab and Google Cloud tutorials.\n",[232,1264,2309,9,1016],"GKE",{"slug":2311,"featured":6,"template":700},"google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab","content:en-us:blog:google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab.yml","Google Cloud Integrations For Secure Cloud Run Deployments At Gitlab","en-us/blog/google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab.yml","en-us/blog/google-cloud-integrations-for-secure-cloud-run-deployments-at-gitlab",{"_path":2317,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2318,"content":2324,"config":2329,"_id":2331,"_type":13,"title":2332,"_source":15,"_file":2333,"_stem":2334,"_extension":18},"/en-us/blog/guide-to-fulfilling-soc-2-security-requirements-with-gitlab",{"title":2319,"description":2320,"ogTitle":2319,"ogDescription":2320,"noIndex":6,"ogImage":2321,"ogUrl":2322,"ogSiteName":685,"ogType":686,"canonicalUrls":2322,"schema":2323},"Guide to fulfilling SOC 2 security requirements with GitLab","Understand the application security features in the GitLab DevSecOps platform that map to System and Organization Controls 2 requirements.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099576/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_1172300481_IGPi3TS4VzFgcqhvEdBlR_1750099575518.jpg","https://about.gitlab.com/blog/guide-to-fulfilling-soc-2-security-requirements-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Guide to fulfilling SOC 2 security requirements with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-01-22\",\n      }",{"title":2319,"description":2320,"authors":2325,"heroImage":2321,"date":2326,"body":2327,"category":830,"tags":2328},[1444],"2025-01-22","For businesses that handle sensitive customer information, achieving SOC 2\n(System and Organization Controls 2) compliance is not just a good practice\n— it's often a necessity. SOC 2 is a rigorous auditing standard developed by\nthe American Institute of Certified Public Accountants that assesses a\nservice organization's controls related to security, availability,\nprocessing integrity, confidentiality, and privacy.\n\n\nWhile SOC 2 is not legally mandated, it has become increasingly important,\nin part due to breaches consistently seen in news headlines. Obtaining SOC 2\ncompliance allows customers to build trust with service organizations\nbecause they know their data is being properly stored and security controls\nhave been assessed by a third party.\n\n\nIn this guide, we'll review the requirements for obtaining SOC 2 compliance\nand how GitLab can help your organization meet the highest standards for\napplication security.\n\n\n## What requirements are set by SOC 2\n\n\nThe compliance process involves an audit by an independent auditor who\nevaluates the design and operating effectiveness of an organization's\ncontrols. This process can be very costly, and many organizations are not\nsufficiently prepared before an audit. With the SOC 2 audit process\ntypically taking close to a year, it is important to establish an efficient\npre-audit process.\n\n\nTo obtain SOC 2 compliance, an organization must meet requirements based on\nthe Trust Services Criteria:\n\n\n| Criteria | Requirements |\n\n| :---- | :---- |\n\n| Security | - Implement controls to protect against unauthorized access\n\u003Cbr> - Establish procedures for identifying and mitigating risks\u003Cbr> - Set\nup systems for detecting and addressing security incidents |\n\n| Availability | - Ensure systems are accessible for operation as agreed\u003Cbr>\n- Monitor current usage and capacity \u003Cbr> - Identify and address\nenvironmental threats that could affect system availability |\n\n| Process integrity | - Maintain accurate records of system inputs and\noutputs \u003Cbr> - Implement procedures to quickly identify and correct system\nerrors \u003Cbr> - Define processing activities to ensure products and services\nmeet specifications |\n\n| Confidentiality | - Identify and protect confidential information \u003Cbr> -\nEstablish policies for data retention periods \u003Cbr> - Implement secure\nmethods for destroying confidential data after retention periods expire |\n\n| Privacy | - Obtain consent before collecting sensitive personal\ninformation \u003Cbr> - Communicate privacy policies clearly and in plain\nlanguage \u003Cbr> - Collect data only through legal means and from reliable\nsources |\n\n\u003Cbr>\n\n\nNote that these requirements are not one-time achievements, but rather a\ncontinuous process. Auditors will require control effectiveness over time.\n\n\n## How to achieve and maintain the security requirements\n\n\nGitLab provides several features off the board to get you started with\nassuring SOC 2 security needs are met:\n\n\n| Security Requirement | Addressing Feature |\n\n| :---- | :---- |\n\n| Implement controls to protect against unauthorized access | - Confidential\nIssues and Merge Requests \u003Cbr> - Custom Roles and Granular Permissions \u003Cbr>\n- Security Policies \u003Cbr> - Verified Commit \u003Cbr> - Signed Container Images\n\u003Cbr> - CodeOwners \u003Cbr> - Protected Branches |\n\n| Set up systems for detecting and addressing security incidents | -\nVulnerability Scanning \u003Cbr> - Merge Request Security Widget \u003Cbr> -\nVulnerability Insights Compliance Center \u003Cbr> - Audit Events \u003Cbr> -\nVulnerability Report Dependency List \u003Cbr> - AI: Vulnerability Explanation\n\u003Cbr> - AI: Vulnerability Resolution |\n\n| Establish procedures for identifying and mitigating risks | All the above\ntools can be used by a security team to establish a procedure around what to\ndo when security vulnerabilities are identified and how they are mitigated.\n|\n\n\u003Cbr>\n\nLet’s go through each section and highlight the security features that\naddress these requirements. Note that a [GitLab Ultimate\nsubscription](https://about.gitlab.com/free-trial/) and the correct Role and\nPermissions are required to access many of the features listed. Be sure to\ncheck out the appropriate documentation for more information.\n\n\n## Implement controls to protect against unauthorized access\n\n\nImplementing robust access controls is essential for protecting an\norganization's assets, ensuring regulatory compliance, maintaining\noperational continuity, and fostering trust. GitLab allows you to implement\ncontrols to follow the [principle of least\nprivilege](https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab/),\nsecuring against unauthorized access. I will briefly cover:\n\n\n* [Security policies](#security-policies)  \n\n* [Custom roles and granular\npermissions](#custom-roles-and-granular-permissions)  \n\n* [Branch protections and CodeOwners](#branch-protections-and-codeowners)  \n\n* [Verified commits](#verified-commits)\n\n\n### Security policies\n\n\nGitLab's security policies, known as guardrails, enable security and\ncompliance teams to implement consistent controls across their organization,\nhelping prevent security incidents, maintain compliance standards, and\nreduce risk by automatically enforcing security best practices at scale.\n\n\n![Merge request approval policy in\naction](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/merge_request_approval_policy_aHR0cHM6_1750099596925.png)\n\n\n\u003Ccenter>\u003Ci>Merge request approval policy in action\u003C/i>\u003C/center>\u003Cbr>\n\n\nThe following policy types are available:\n\n\n* Scan execution policy: Enforce security scans, either as part of the\npipeline or on a specified schedule  \n\n* Merge request approval policy: Enforce project-level settings and approval\nrules based on scan results  \n\n* Pipeline execution policy: Enforce CI/CD jobs as part of project\npipelines  \n\n* Vulnerability management policy: Automate vulnerability management\nworkflows\n\n\nHere is an example of ensuring compliance with the pipeline execution\npolicy:\n\n\n1. Create a project that houses multiple compliance jobs. An example of a\njob can be to check permissions of files that are deployed. These jobs\nshould be generic enough that they can be applied to multiple applications.\n\n2. Limit the project's permissions to only security/compliance officers;\ndon’t allow developers to remove jobs. This allows for separation of duties.\n\n3. Inject the compliance jobs in batch to the projects where they are\nrequired. Force them to run no matter what, but allow approval from team\nlead to not block development. This will ensure compliance jobs are always\nrun and cannot be removed by developers, and that your environment remains\ncompliant.\n\n\n> ##### Learn how to create security policies with our [security policy\ndocumentation](https://docs.gitlab.com/ee/user/application_security/policies/).\n\n\n### Custom roles and granular permissions\n\n\nCustom permissions in GitLab allow organizations to create fine-grained\naccess controls beyond the standard role-based permissions, providing\nbenefits such as:\n\n\n* more precise access control  \n\n* better security compliance  \n\n* reduced risk of accidental access  \n\n* streamlined user management  \n\n* support for complex organizational structures\n\n\n![GitLab custom\nroles](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/custom_roles_aHR0cHM6_1750099596926.png)\n\n\n\u003Ccenter>\u003Ci>Roles and permissions settings, including custom\nroles\u003C/i>\u003C/center>\n\n\n> ##### Learn how to create custom roles with granular permissions using our\n[custom role\ndocumentation](https://docs.gitlab.com/ee/user/custom_roles.html).\n\n\n### Branch protections and CodeOwners\n\n\nGitLab helps you further control who can change your code using two key\nfeatures:\n\n* Branch Protection, which lets you set rules about who can update specific\nbranches – like requiring approval before merging changes.\n\n* Code Ownership, which automatically finds the right people to review code\nchanges by matching files to their designated owners.\n\n\nTogether, these features help keep your code secure and high-quality by\nmaking sure the right people review and approve changes.\n\n\n![Protected\nbranches](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/protected_branches_aHR0cHM6_1750099596928.png)\n\n\n\u003Ccenter>\u003Ci>Protected branch settings\u003C/i>\u003C/center>\n\n\n> ##### Learn how to create protected branches along with CodeOwners using\n[protected\nbranch](https://docs.gitlab.com/ee/user/project/repository/branches/protected.html)\nand [codeowner](https://docs.gitlab.com/ee/user/project/codeowners/)\ndocumentation.\n\n\n### Verified commits\n\n\nWhen you sign your commits digitally, you prove they really came from you,\nnot someone pretending to be you. Think of a digital signature like a unique\nstamp that only you can create. When you upload your public GPG key to\nGitLab, it can check this stamp. If the stamp matches, GitLab marks your\ncommit as `Verified`. You can then set up rules to reject commits that\naren't signed, or block all commits from users who haven't verified their\nidentity.\n\n\n![Commit signed with verified\nsignature](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/signed_commit_aHR0cHM6_1750099596929.png)\n\n\n\u003Ccenter>\u003Ci>Commit signed with verified signature\u003C/i>\u003C/center>\u003Cbr>\n\n\nCommits can be signed with:\n\n\n* SSH key  \n\n* GPG key  \n\n* Personal x.509 certificate\n\n\n> ##### Learn more about verified commits with our [signed commits\ndocumentation](https://docs.gitlab.com/ee/user/project/repository/signed_commits/).\n\n\n## Set up systems for detecting and addressing security incidents\n\n\nSetting up systems for detecting and addressing security incidents is vital\nfor maintaining a robust security posture, ensuring regulatory compliance,\nminimizing potential damages, and enabling organizations to respond\neffectively to the ever-evolving threat landscape.\n\n\nGitLab provides security scanning and vulnerability management for the\ncomplete application lifecycle. I will briefly cover:\n\n\n* [Security scanning and vulnerability\nmanagement](#security-scanning-and-vulnerability-management)  \n\n* [Software bill of materials](#software-bill-of-materials)  \n\n* [System auditing and security posture\nreview](#system-auditing-and-security-posture-review)\n\n* [Compliance and security posture\noversight](#compliance-and-security-posture-oversight)\n\n\n### Security scanning and vulnerability management\n\n\nGitLab provides a variety of different security scanners that cover the\ncomplete lifecycle of your application:\n\n\n* Static Application Security Testing (SAST)  \n\n* Dynamic Application Security Testing (DAST)\n\n* Container Scanning  \n\n* Dependency Scanning  \n\n* Infrastructure as Code (IaC) Scanning  \n\n* Coverage-guided Fuzzing\n\n* Web API Fuzzing\n\n\nThese scanners can be added to your pipeline via the use of templates. For\nexample, to run SAST and dependency scanning jobs in the test stage, simply\nadd the following to your .gitlab-ci.yml:\n\n\n```yaml  \n\nstages:  \n   - test\n\ninclude:  \n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml  \n  - template: Jobs/SAST.gitlab-ci.yml  \n``` \n\n\nThese jobs are fully configurable via environment variables and using GitLab\njob syntax. Once a pipeline kicks off, the security scanners run and detect\nvulnerabilities in the diff between the current branch and the target\nbranch. The vulnerability can be seen in a merge request (MR), providing\ndetailed oversight before the code is merged to the target branch. The MR\nwill provide the following information on a vulnerability:\n\n\n* description  \n\n* status  \n\n* severity  \n\n* evidence  \n\n* identifiers  \n\n* URL (if applicable)  \n\n* request/response (if applicable)  \n\n* reproduction assets (if applicable)  \n\n* training (if applicable)  \n\n* code flow (if using advanced SAST)\n\n\n![MR view of introduced\nvulnerability](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/no_sql_injection_vulnerability_mr_view_aHR0cHM6_1750099596931.png)\n\n\n\u003Ccenter>\u003Ci>MR view of introduced vulnerability\u003C/i>\u003C/center>\u003Cbr>\n\n\nDevelopers can use this data to remediate vulnerabilities without slowing\ndown security team workflows. Developers can dismiss a vulnerability with\nreasoning, speeding up the review process, or they can create a confidential\nissue to track the vulnerability.\n\n\nIf the code in an MR is merged to the default (usually production-level)\nbranch, then the vulnerability report is populated with the security scanner\nresults. These results can be used by security teams to manage and triage\nthe vulnerabilities found in production.\n\n\n![Vulnerability report with Batch Status\nsetting](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/vulnerability_report_aHR0cHM6_1750099596936.png)\n\n\n\u003Ccenter>\u003Ci>Vulnerability report with Batch Status setting\u003C/i>\u003C/center>\u003Cbr>\n\n\nWhen clicking on a vulnerability description within the vulnerability\nreport, you are provided with the vulnerability page, which contains the\nsame vulnerability data as the MR, allowing for a single source of truth\nwhen assessing impact and performing remediation. From the vulnerability\npage, [GitLab Duo](https://about.gitlab.com/gitlab-duo/) AI features can be\nused to explain the vulnerability and also create an MR to remediate,\nspeeding up resolution time.\n\n\n> ##### Learn more about the security scanners included with GitLab and how\nto manage vulnerabilities in our [application security\ndocumentation](https://docs.gitlab.com/ee/user/application_security/).\n\n\n### Software bill of materials\n\n\nGitLab can create a detailed list of everything your software uses – kind of\nlike an ingredients list for your code. This list, called a software bill of\nmaterials\n([SBOM](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/)),\nshows you all the external code your project depends on, including the parts\nyou directly use and their own dependencies. For each item, you can see\nwhich version you're using, what license it has, and whether it has any\nknown security problems. This helps you keep track of what's in your\nsoftware and spot potential risks.\n\n\n![Group-level dependency list\n(SBOM)](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/sbom_aHR0cHM6_1750099596937.png)\n\n\n\u003Ccenter>\u003Ci>Group-level dependency list (SBOM)\u003C/i>\u003C/center>\n\n\n> ##### Learn how to access and use the dependency list with our [dependency\nlist\ndocumentation](https://docs.gitlab.com/ee/user/application_security/dependency_list/).\n\n\n### System auditing and security posture review\n\n\nGitLab keeps track of everything that happens in your system such as who\nmade changes, what they changed, and when they did it. Think of it like a\nsecurity camera for your code. This record helps you:\n\n\n* spot any suspicious activity  \n\n* show regulators you're following the rules  \n\n* figure out what happened if something goes wrong  \n\n* see how people are using GitLab\n\n\nAll of this information is stored in one place, making it easy to review and\ninvestigate when needed. For example, you can use audit events to track:\n\n\n* who changed the permission level of a particular user for a GitLab\nproject, and when  \n\n* who added a new user or removed a user, and when\n\n\n![Project-level audit\nevents](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/audit_events_aHR0cHM6_1750099596938.png)\n\n\n\u003Ccenter>\u003Ci>Project-level audit events\u003C/i>\u003C/center>\n\n\n> ##### Learn more about audit events, see the [audit events\ndocumentation](https://docs.gitlab.com/ee/user/compliance/audit_events.html).\n\n\n## Compliance and security posture oversight\n\n\nGitLab's Security Dashboard works like a control room that shows you all\nyour security risks in one place. Instead of checking different security\ntools separately, you can see all their findings together on one screen.\nThis makes it easy to spot and fix security problems across all your\nprojects.\n\n\n![Group-level Security\nDashboard](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099597/Blog/Content%20Images/Blog/Content%20Images/security_dashboard_aHR0cHM6_1750099596939.png)\n\n\u003Ccenter>\u003Ci>Group-level security dashboard\u003C/i>\u003C/center>\n\n\n> ##### Learn more about security dashboards with our [security dashboard\ndocumentation](https://docs.gitlab.com/ee/user/application_security/security_dashboard/).\n\n\n## Establish procedures for identifying and mitigating risks\n\n\nVulnerabilities go through a specific lifecycle. For example, a part of the\nprocedure can be to require approval for any vulnerable code to be merged to\nprotected branches using security policies. Then the procedure can state\nthat vulnerable code detected in production must be prioritized, assessed,\nremediated, and then validated: \n\n\n* The criteria for prioritization can be by the severity of the\nvulnerability provided by GitLab scanners.  \n\n* The assessment can be done using exploitation details provided by the AI:\nVulnerability Explanation.  \n\n* Once the vulnerability is remediated, then it can be validated using\nbuilt-in GitLab regression tests and scanners.\n\n\nWhile every organization's needs are different, leveraging GitLab as a\nplatform, risks can be quickly identified and addressed with reduced risk\nwhen compared to using a sprawl of disparate tools.\n\n\n### Best practices for SOC 2 compliance\n\n\n* Establish a strong security culture: Foster a culture of security\nawareness and accountability throughout your organization.  \n\n* Document everything: Maintain thorough documentation of policies,\nprocedures, and controls.  \n\n* Automate where possible: Use automation tools to streamline compliance\nprocesses and reduce errors.  \n\n* Communicate effectively: Keep stakeholders informed about your compliance\nefforts.  \n\n* Seek expert guidance: Consider partnering with a qualified consultant to\nassist with your SOC 2 journey.\n\n\nAchieving SOC 2 compliance is a significant undertaking, but the benefits\nare undeniable. By demonstrating your commitment to application security and\noperational excellence, you can build trust with customers, enhance your\nreputation, and gain a competitive edge in the marketplace.\n\n\n## Read more\n\n\nTo learn more about GitLab and how we can help achieve SOCv2 compliance\nwhile enhancing your security posture, check out the following resources:\n\n\n* [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/)  \n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab Application Security\nDocumentation](https://docs.gitlab.com/ee/user/application_security/)  \n\n* [GitLab DevSecOps Tutorial\nProject](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\n",[9,830,495,696,694],{"slug":2330,"featured":90,"template":700},"guide-to-fulfilling-soc-2-security-requirements-with-gitlab","content:en-us:blog:guide-to-fulfilling-soc-2-security-requirements-with-gitlab.yml","Guide To Fulfilling Soc 2 Security Requirements With Gitlab","en-us/blog/guide-to-fulfilling-soc-2-security-requirements-with-gitlab.yml","en-us/blog/guide-to-fulfilling-soc-2-security-requirements-with-gitlab",{"_path":2336,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2337,"content":2343,"config":2349,"_id":2351,"_type":13,"title":2352,"_source":15,"_file":2353,"_stem":2354,"_extension":18},"/en-us/blog/hosting-vuejs-apps-using-gitlab-pages",{"title":2338,"description":2339,"ogTitle":2338,"ogDescription":2339,"noIndex":6,"ogImage":2340,"ogUrl":2341,"ogSiteName":685,"ogType":686,"canonicalUrls":2341,"schema":2342},"How to host VueJS apps using GitLab Pages","Follow this tutorial, including detailed configuration guidance, to quickly get your application up and running for free.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683489/Blog/Hero%20Images/hosting.png","https://about.gitlab.com/blog/hosting-vuejs-apps-using-gitlab-pages","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to host VueJS apps using GitLab Pages\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sophia Manicor\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2023-09-13\",\n      }",{"title":2338,"description":2339,"authors":2344,"heroImage":2340,"date":2346,"body":2347,"category":785,"tags":2348},[2345,847],"Sophia Manicor","2023-09-13","\nIf you use VueJS to build websites, then you can host your website for free with GitLab Pages. This short tutorial walks you through a simple way to host and deploy your VueJS applications using GitLab CI/CD and GitLab Pages.\n\n## Prequisites\n- A VueJS application\n- Working knowledge of GitLab CI\n- 5 minutes\n\n## Setting up your VueJS application\n\n1) Install vue-cli.\n\n```bash\nnpm install -g @vue/cli\n# OR\nyarn global add @vue/cli\n```\nYou can check you have the right version of Vue with:\n\n```bash\nvue --version\n```\n\n2) Create your application using:\n\n```bash\nvue create name-of-app\n```\n\nWhen successfully completed, you will have a scaffolding of your VueJS application.\n\n## Setting up .gitlab-ci.yml for GitLab Pages\nBelow is the [GitLab CI configuration](https://gitlab.com/demos/applications/vuejs-gitlab-pages/-/blob/main/.gitlab-ci.yml) necessary to deploy to GitLab Pages. Put this file into your root project. GitLab Pages always deploys your website from a specific folder called `public`.\n\n```yaml\nimage: \"node:16-alpine\"\n\nstages:\n  - build\n  - test\n  - deploy\n\nbuild:\n  stage: build\n  script:\n    - yarn install --frozen-lockfile --check-files --non-interactive\n    - yarn build\n  artifacts:\n    paths:\n      - public\n\npages:\n  stage: deploy\n  script:\n    - echo 'Pages deployment job'\n  artifacts:\n    paths:\n      - public\n  only:\n    - main\n\n```\n\n## Vue config (vue.config.js)\nIn Vue, the artifacts are built in a folder called dist, in order for GitLab to deploy to Pages, we need to change the path of the artifacts. One way to do this is by changing the [Vue config file](https://gitlab.com/demos/applications/vuejs-gitlab-pages/-/blob/main/vue.config.js), `vue.config.js`.\n\n```\nconst { defineConfig } = require('@vue/cli-service')\n\nfunction publicPath () {\n  if (process.env.CI_PAGES_URL) {\n    return new URL(process.env.CI_PAGES_URL).pathname\n  } else {\n    return '/'\n  }\n}\n\nmodule.exports = defineConfig({\n  transpileDependencies: true,\n  publicPath: publicPath(),\n  outputDir: 'public'\n})\n```\n\nHere we have set `outputDir` to `public` so that GitLab will pick up the build artifacts and deploy to Pages. Another important piece when creating this configuration file is to change the `publicPath`, which is the base URL your application will be deployed at. In this case, we have create a function `publicPath()` that checks if the CI_PAGES_URL environment variable is set and returns the correct base URL.\n\n## Run GitLab CI\n\n![vuejs-gitlab-pages-pipeline](https://about.gitlab.com/images/blogimages/2023-05-11-hosting-vuejs-apps-using-gitlab-pages/vuejs-gitlab-pages-pipeline.png){: .shadow}\n\n\n## Check Pages to get your URL\n\n![gitlab-pages-domain](https://about.gitlab.com/images/blogimages/2023-05-11-hosting-vuejs-apps-using-gitlab-pages/gitlab-page-domain.png){: .shadow}\n\nVoila! You have set up a VueJS project with a fully functioning CI/CD pipeline. Enjoy your VueJS application hosted by GitLab Pages!\n\n## References\n- [https://cli.vuejs.org/guide/installation.html](https://cli.vuejs.org/guide/installation.html)\n- [https://cli.vuejs.org/guide/creating-a-project.html](https://cli.vuejs.org/guide/creating-a-project.html)\n- [https://gitlab.com/demos/applications/vuejs-gitlab-pages](https://gitlab.com/demos/applications/vuejs-gitlab-pages)\n\n",[108,9,787,788],{"slug":2350,"featured":6,"template":700},"hosting-vuejs-apps-using-gitlab-pages","content:en-us:blog:hosting-vuejs-apps-using-gitlab-pages.yml","Hosting Vuejs Apps Using Gitlab Pages","en-us/blog/hosting-vuejs-apps-using-gitlab-pages.yml","en-us/blog/hosting-vuejs-apps-using-gitlab-pages",{"_path":2356,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2357,"config":2360,"content":2362,"_id":2369,"_type":13,"title":2370,"_source":15,"_file":2371,"_stem":2372,"_extension":18},"/en-us/blog/how-gitlab-duo-agent-platform-transforms-dataops",{"title":2358,"description":2359},"How GitLab Duo Agent Platform transforms DataOps","Explore how to turn hours of manual coding into minutes of automated generation with this comprehensive dbt model creation walkthrough.",{"slug":2361,"featured":90,"template":700},"how-gitlab-duo-agent-platform-transforms-dataops",{"title":2358,"description":2359,"authors":2363,"heroImage":2365,"date":2366,"category":718,"tags":2367,"body":2368},[2364],"Dennis van Rooijen","blog/hero%20images/workflow_1800x945.png","2025-09-16",[694,9,696],"Creating dbt models manually is a tedious process that can consume hours of a data engineer's time. Especially when no (big) business transformations are made, it is not the most attractive part of an engineer's work with data.\n\nBut what if you could automate this entire process? In this walkthrough, I'll show you exactly how [GitLab Duo Agent Platform](https://about.gitlab.com/gitlab-duo/agent-platform/) can generate comprehensive dbt models in just minutes, complete with proper structure, tests, and documentation.\n\n## What we're building\n\nOur marketing team wants to effectively manage and optimize advertising investments. One of the advertising platforms is Reddit, so, therefore, we are extracting data from the Reddit Ads API to our enterprise [Data Platform](https://handbook.gitlab.com/handbook/enterprise-data/platform/) Snowflake. At GitLab, we have three layers of storage:\n\n1. `raw` layer - first landing point for unprocessed data from external sources; not ready for business use\n2. `prep` layer - first transformation layer with source models; still not ready for general business use\n3. `prod` layer - final transformed data ready for business use and Tableau reporting\n\n![Chart of storage layers](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758030995/zo7vespktzfdtdtiauz7.png)\n\nFor this walkthrough, data has already landed in the raw layer by our extraction solution Fivetran, and we'll generate dbt models that handle the data through the `prep` layer to the `prod` layer.\n\nWithout having to write a single line of dbt code ourselves, by the end of the walkthrough we will have:\n\n- **Source models** in the prep layer\n- **Workspace models** in the prod layer\n- **Complete dbt configurations** for all 13 tables (which includes 112 columns) in the Reddit Ads dataset\n- **Test queries** to validate the outcomes\n\nThe entire process will take less than 10 minutes, compared to the hours it would typically require manually. Here are the steps to follow:\n\n## 1. Prepare the data structure\n\nBefore GitLab Duo can generate our models, it needs to understand the complete table structure. The key is running a query against Snowflake's information schema, because we are currently investigating how to connect GitLab Duo via Model Context Protocol ([MCP](https://about.gitlab.com/topics/ai/model-context-protocol/)) to our Snowflake instance:\n\n```sql\nSELECT \n    table_name,\n    column_name,\n    data_type,\n    is_nullable,\n    CASE \n        WHEN is_nullable = 'NO' THEN 'PRIMARY_KEY'\n        ELSE NULL \n    END as key_type\nFROM raw.information_schema.columns\nWHERE table_schema = 'REDDIT_ADS'\nORDER BY table_name, ordinal_position;\n```\n\nThis query captures:\n\n- All table and column names\n- Data types for proper model structure\n- Nullable constraints\n- Primary key identification (non-nullable columns in this dataset)\n\n**Pro tip:** In the Reddit Ads dataset, all non-nullable columns serve as primary keys — a pattern. I validated by checking tables like `ad_group`, which has two non-nullable columns (`account_id` and `id`) that are both marked as primary keys. Running this query returned 112 rows of metadata that I exported as a CSV file for model generation. While this manual step works well today, we're investigating a direct GitLab Duo integration with our Data Platform via MCP to automate this process entirely.\n\n## 2. Set up GitLab Duo\n\nThere are two ways to interact with [GitLab Duo](https://docs.gitlab.com/user/get_started/getting_started_gitlab_duo/):\n\n1. **Web UI chat function**\n2. **Visual Studio Code plugin**\n\nI chose the VS Code plugin because I can run the dbt models locally to test them.\n\n## 3. Enter the 'magic' prompt\n\nHere's the exact prompt I used to generate all the dbt code:\n\n```yaml\nCreate dbt models for all the tables in the file structure.csv.\n\nI want to have the source models created, with a filter that dedupes the data based on the primary key. Create these in a new folder reddit_ads.\nI want to have workspace models created and store these in the workspace_marketing schema.\n\nTake this MR as example: [I've referenced to previous source implementation]. Here is the same done for Source A, but now it needs to be done for Reddit Ads. \n\nPlease check the dbt style guide when creating the code: https://handbook.gitlab.com/handbook/enterprise-data/platform/dbt-guide/\n```\n\nKey elements that made this prompt effective:\n\n- **Clear specifications** for both source and workspace models.\n- **Reference example** from a previous similar merge request.\n- **Style guide reference** to ensure code quality and consistency.\n- **Specific schema targeting** for proper organization.\n\n## 4. GitLab Duo's process\n\nAfter submitting the prompt, GitLab Duo got to work. The entire generation process took a few minutes, during which GitLab Duo:\n\n1. **Read and analyzed** the CSV input file.\n2. **Examined table structures** from the metadata.\n3. **Referenced our dbt style guide** for coding standards.\n4. **Took similar merge request into account** to properly structure.\n5. **Generated source models** for all 13 tables.\n6. **Created workspace models** for all 13 tables.\n7. **Generated supporting dbt files**:\n   - `sources.yml` configuration.\n   - `schema.yml` files with tests and documentation.\n   - Updated `dbt_project.yml` with schema references.\n\n## The results\n\nThe output was remarkable:\n\n- **1 modified file:** dbt_project.yml (added reddit_ads schema configuration)\n- **29 new files:**\n  - **26 dbt models** (13 source + 13 workspace)\n  - **3 YAML files**\n- **Nearly 900 lines of code** generated automatically\n- **Built-in data tests,** including unique constraints on primary key columns\n- **Generic descriptions** for all models and columns\n- **Proper deduplication logic** in source models\n- **Clean, consistent code structure** following the GitLab dbt style guide\n\n```yaml\ntransform/snowflake-dbt/\n├── dbt_project.yml                                                    [MODIFIED]\n└── models/\n    ├── sources/\n    │   └── reddit_ads/\n    │       ├── reddit_ads_ad_group_source.sql                        [NEW]\n    │       ├── reddit_ads_ad_source.sql                              [NEW]\n    │       ├── reddit_ads_business_account_source.sql                [NEW]\n    │       ├── reddit_ads_campaign_source.sql                        [NEW]\n    │       ├── reddit_ads_custom_audience_history_source.sql         [NEW]\n    │       ├── reddit_ads_geolocation_source.sql                     [NEW]\n    │       ├── reddit_ads_interest_source.sql                        [NEW]\n    │       ├── reddit_ads_targeting_community_source.sql             [NEW]\n    │       ├── reddit_ads_targeting_custom_audience_source.sql       [NEW]\n    │       ├── reddit_ads_targeting_device_source.sql                [NEW]\n    │       ├── reddit_ads_targeting_geolocation_source.sql           [NEW]\n    │       ├── reddit_ads_targeting_interest_source.sql              [NEW]\n    │       ├── reddit_ads_time_zone_source.sql                       [NEW]\n    │       ├── schema.yml                                            [NEW]\n    │       └── sources.yml                                           [NEW]\n    └── workspaces/\n        └── workspace_marketing/\n            └── reddit_ads/\n                ├── schema.yml                                        [NEW]\n                ├── wk_reddit_ads_ad.sql                              [NEW]\n                ├── wk_reddit_ads_ad_group.sql                        [NEW]\n                ├── wk_reddit_ads_business_account.sql                [NEW]\n                ├── wk_reddit_ads_campaign.sql                        [NEW]\n                ├── wk_reddit_ads_custom_audience_history.sql         [NEW]\n                ├── wk_reddit_ads_geolocation.sql                     [NEW]\n                ├── wk_reddit_ads_interest.sql                        [NEW]\n                ├── wk_reddit_ads_targeting_community.sql             [NEW]\n                ├── wk_reddit_ads_targeting_custom_audience.sql       [NEW]\n                ├── wk_reddit_ads_targeting_device.sql                [NEW]\n                ├── wk_reddit_ads_targeting_geolocation.sql           [NEW]\n                ├── wk_reddit_ads_targeting_interest.sql              [NEW]\n                └── wk_reddit_ads_time_zone.sql                       [NEW]\n```\n\n### Sample generated code\n\nHere's an example of the generated code quality. For the `time_zone` table, GitLab Duo created:\n\n**Prep Layer Source Model**\n\n```sql\nWITH source AS (\n  SELECT *\n  FROM {{ source('reddit_ads','time_zone') }}\n  QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY _fivetran_synced DESC) = 1\n),\n\nrenamed AS (\n  SELECT\n    id::VARCHAR                               AS time_zone_id,\n    code::VARCHAR                             AS time_zone_code,\n    dst_offset::NUMBER                        AS time_zone_dst_offset,\n    is_dst_active::BOOLEAN                    AS is_time_zone_dst_active,\n    name::VARCHAR                             AS time_zone_name,\n    offset::NUMBER                            AS time_zone_offset,\n    _fivetran_synced::TIMESTAMP               AS fivetran_synced_at\n  FROM source\n)\n\nSELECT * FROM renamed\n```\n\n**Schema.yml**\n\n```yaml\nmodels:\n  - name: reddit_ads_time_zone_source\n    description: Time zone data from Reddit Ads system\n    columns:\n      - name: time_zone_id\n        description: Unique identifier for time zone records\n        data_tests:\n          - unique\n          - not_null\n      - name: time_zone_code\n        description: Code for the time zone\n      - name: time_zone_dst_offset\n        description: Daylight saving time offset for the time zone\n      - name: is_time_zone_dst_active\n        description: Flag indicating if daylight saving time is active\n      - name: time_zone_name\n        description: Name of the time zone\n      - name: time_zone_offset\n        description: Offset for the time zone\n      - name: fivetran_synced_at\n        description: Timestamp when the record was last synced by Fivetran\n```\n\n**Source.yml**\n\n```yaml\nsources:\n  - name: reddit_ads\n    database: RAW\n    schema: reddit_ads\n    loaded_at_field: _fivetran_synced\n    loader: fivetran\n    description: Reddit Ads data\n\n    quoting:\n      database: true\n      schema: false\n      identifier: false\n\n    tables:\n      - name: time_zone\n```\n\n**Workspace Model**\n\n```sql\nWITH source AS (\n  SELECT *\n  FROM {{ ref('reddit_ads_time_zone_source') }}\n)\n\nSELECT * FROM source\n```\n\n## 5. Quality validation\n\nNow that the code looks good, I pushed it to the MR and executed [CI test pipeline](https://handbook.gitlab.com/handbook/enterprise-data/platform/ci-jobs/#build_changes) to test the code and validate the outcome. I asked GitLab Duo to create a validation query:\n\n```yaml\nCreate a test query to test the row counts between the raw layer and the workspace layer. Keep in mind that we do deduplication, so we can compare both using distinct on the primary keys.\n```\n\nThe AI generated a comprehensive validation query that:\n\n- Compared row counts between raw and workspace layers.\n- Accounted for deduplication logic.\n- Tested all 13 tables.\n- Calculated data retention percentages.\n\n\u003Cdetails>\n\u003Csummary>Generated SQL Test query\u003C/summary>\n\n```sql\n-- Reddit Ads Row Count Validation Test\n-- Compares distinct counts between RAW layer and WORKSPACE_MARKETING layer\n-- Accounts for deduplication logic in source models\n\nWITH raw_counts AS (\n  -- Single primary key tables\n  SELECT 'ad' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.AD\n  \n  UNION ALL\n  \n  SELECT 'business_account' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.BUSINESS_ACCOUNT\n  \n  UNION ALL\n  \n  SELECT 'campaign' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.CAMPAIGN\n  \n  UNION ALL\n  \n  SELECT 'custom_audience_history' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.CUSTOM_AUDIENCE_HISTORY\n  \n  UNION ALL\n  \n  SELECT 'geolocation' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.GEOLOCATION\n  \n  UNION ALL\n  \n  SELECT 'interest' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.INTEREST\n  \n  UNION ALL\n  \n  SELECT 'time_zone' AS table_name, COUNT(DISTINCT id) AS raw_count\n  FROM RAW.REDDIT_ADS.TIME_ZONE\n  \n  -- Composite primary key tables\n  UNION ALL\n  \n  SELECT 'ad_group' AS table_name, COUNT(DISTINCT CONCAT(account_id, '|', id)) AS raw_count\n  FROM RAW.REDDIT_ADS.AD_GROUP\n  \n  UNION ALL\n  \n  SELECT 'targeting_community' AS table_name, COUNT(DISTINCT CONCAT(ad_group_id, '|', community_id)) AS raw_count\n  FROM RAW.REDDIT_ADS.TARGETING_COMMUNITY\n  \n  UNION ALL\n  \n  SELECT 'targeting_custom_audience' AS table_name, COUNT(DISTINCT CONCAT(ad_group_id, '|', custom_audience_id)) AS raw_count\n  FROM RAW.REDDIT_ADS.TARGETING_CUSTOM_AUDIENCE\n  \n  UNION ALL\n  \n  SELECT 'targeting_device' AS table_name, COUNT(DISTINCT _fivetran_id) AS raw_count\n  FROM RAW.REDDIT_ADS.TARGETING_DEVICE\n  \n  UNION ALL\n  \n  SELECT 'targeting_geolocation' AS table_name, COUNT(DISTINCT CONCAT(ad_group_id, '|', geolocation_id)) AS raw_count\n  FROM RAW.REDDIT_ADS.TARGETING_GEOLOCATION\n  \n  UNION ALL\n  \n  SELECT 'targeting_interest' AS table_name, COUNT(DISTINCT CONCAT(ad_group_id, '|', interest_id)) AS raw_count\n  FROM RAW.REDDIT_ADS.TARGETING_INTEREST\n),\n\nworkspace_counts AS (\n  -- Workspace layer counts using primary keys from schema.yml\n  SELECT 'ad' AS table_name, COUNT(DISTINCT ad_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_AD\n  \n  UNION ALL\n  \n  SELECT 'business_account' AS table_name, COUNT(DISTINCT business_account_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_BUSINESS_ACCOUNT\n  \n  UNION ALL\n  \n  SELECT 'campaign' AS table_name, COUNT(DISTINCT campaign_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_CAMPAIGN\n  \n  UNION ALL\n  \n  SELECT 'custom_audience_history' AS table_name, COUNT(DISTINCT custom_audience_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_CUSTOM_AUDIENCE_HISTORY\n  \n  UNION ALL\n  \n  SELECT 'geolocation' AS table_name, COUNT(DISTINCT geolocation_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_GEOLOCATION\n  \n  UNION ALL\n  \n  SELECT 'interest' AS table_name, COUNT(DISTINCT interest_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_INTEREST\n  \n  UNION ALL\n  \n  SELECT 'time_zone' AS table_name, COUNT(DISTINCT time_zone_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TIME_ZONE\n  \n  -- Composite primary key tables\n  UNION ALL\n  \n  SELECT 'ad_group' AS table_name, COUNT(DISTINCT CONCAT(ad_group_account_id, '|', ad_group_id)) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_AD_GROUP\n  \n  UNION ALL\n  \n  SELECT 'targeting_community' AS table_name, COUNT(DISTINCT CONCAT(targeting_community_ad_group_id, '|', targeting_community_id)) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TARGETING_COMMUNITY\n  \n  UNION ALL\n  \n  SELECT 'targeting_custom_audience' AS table_name, COUNT(DISTINCT CONCAT(targeting_custom_audience_ad_group_id, '|', targeting_custom_audience_id)) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TARGETING_CUSTOM_AUDIENCE\n  \n  UNION ALL\n  \n  SELECT 'targeting_device' AS table_name, COUNT(DISTINCT targeting_device_fivetran_id) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TARGETING_DEVICE\n  \n  UNION ALL\n  \n  SELECT 'targeting_geolocation' AS table_name, COUNT(DISTINCT CONCAT(targeting_geolocation_ad_group_id, '|', targeting_geolocation_id)) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TARGETING_GEOLOCATION\n  \n  UNION ALL\n  \n  SELECT 'targeting_interest' AS table_name, COUNT(DISTINCT CONCAT(targeting_interest_ad_group_id, '|', targeting_interest_id)) AS workspace_count\n  FROM REDDIT_DBT_MODEL_GENERATION_PROD.WORKSPACE_MARKETING.WK_REDDIT_ADS_TARGETING_INTEREST\n)\n\n-- Final comparison with validation results\nSELECT \n  r.table_name,\n  r.raw_count,\n  w.workspace_count,\n  r.raw_count - w.workspace_count AS count_difference,\n  CASE \n    WHEN r.raw_count = w.workspace_count THEN '✅ PASS'\n    WHEN r.raw_count > w.workspace_count THEN '⚠️ RAW > WORKSPACE (Expected due to deduplication)'\n    ELSE '❌ FAIL - WORKSPACE > RAW (Unexpected)'\n  END AS validation_status,\n  ROUND((w.workspace_count::FLOAT / r.raw_count::FLOAT) * 100, 2) AS data_retention_percentage\nFROM raw_counts r\nJOIN workspace_counts w ON r.table_name = w.table_name\nORDER BY r.table_name;\n```\n\n\u003C/details>\n\n![query results table](https://res.cloudinary.com/about-gitlab-com/image/upload/v1758030995/guicjhzwvrz3czwjs3xo.png)\n\nRunning this query showed:\n\n- **Zero differences** in row counts after deduplication\n- **100% data retention** across all tables\n- **All tests passed** successfully\n\n## The bottom line: Massive time savings\n\n- **Traditional approach:** 6-8 hours of manual coding, testing, and debugging\n\n- **GitLab Duo approach:** 6-8 minutes of generation + review time\n\nThis represents a 60x improvement in developer efficiency (from 6-8 hours to 6-8 minutes), while maintaining high code quality.\n\n## Best practices for success\n\nBased on this experience, here are key recommendations:\n\n### Prepare your metadata\n\n- Extract complete table structures including data types and constraints.\n- Identify primary keys and relationships upfront.\n- Export clean, well-formatted CSV input files.\n\n**Note:** By connecting GitLab Duo via MCP to your (meta)data, you could exclude this manual step.\n\n### Provide clear context\n\n- Reference existing example MRs when possible.\n- Specify your coding standards and style guides.\n- Be explicit about folder structure and naming conventions.\n\n### Validate thoroughly\n\n- Always create validation queries for data integrity.\n- Test locally before merging.\n- Run your CI/CD pipeline to catch any issues.\n\n### Leverage AI for follow-up tasks\n\n- Generate test queries automatically.\n- Create documentation templates.\n- Build validation scripts.\n\n## What's next\n\nThis demonstration shows how AI-powered development tools like GitLab Duo are also transforming data engineering workflows. The ability to generate hundreds of lines of production-ready code in minutes —  complete with tests, documentation, and proper structure — represents a fundamental shift in how we approach repetitive development tasks.\n\nBy leveraging AI to handle the repetitive aspects of dbt model creation, data engineers can focus on higher-value activities like data modeling strategy, performance optimization, and business logic implementation.\n\n**Ready to try this yourself?** Start with a small dataset, prepare your metadata carefully, and watch as GitLab Duo transforms hours of work into minutes of automated generation.\n\n> [Trial GitLab Duo Agent Platform today.](https://about.gitlab.com/gitlab-duo/agent-platform/)\n\n## Read more\n\n- [GitLab 18.3: Expanding AI orchestration in software engineering](https://about.gitlab.com/blog/gitlab-18-3-expanding-ai-orchestration-in-software-engineering/)\n- [GitLab Duo Agent Platform Public Beta: Next-gen AI orchestration and more](https://about.gitlab.com/blog/gitlab-duo-agent-platform-public-beta/)\n","content:en-us:blog:how-gitlab-duo-agent-platform-transforms-dataops.yml","How Gitlab Duo Agent Platform Transforms Dataops","en-us/blog/how-gitlab-duo-agent-platform-transforms-dataops.yml","en-us/blog/how-gitlab-duo-agent-platform-transforms-dataops",{"_path":2374,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2375,"content":2381,"config":2386,"_id":2388,"_type":13,"title":2389,"_source":15,"_file":2390,"_stem":2391,"_extension":18},"/en-us/blog/how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code",{"title":2376,"description":2377,"ogTitle":2376,"ogDescription":2377,"noIndex":6,"ogImage":2378,"ogUrl":2379,"ogSiteName":685,"ogType":686,"canonicalUrls":2379,"schema":2380},"Developing GitLab Duo: Secure and thoroughly test AI-generated code","Learn step-by-step how to enhance AI-generated code reliability and security using GitLab Duo and GitLab Pages (includes code samples and prompts).","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097183/Blog/Hero%20Images/Blog/Hero%20Images/blog-hero-banner-1-0178-820x470-fy25_7JlF3WlEkswGQbcTe8DOTB_1750097183481.png","https://about.gitlab.com/blog/how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Developing GitLab Duo: Secure and thoroughly test AI-generated code\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"David O'Regan\"}],\n        \"datePublished\": \"2024-05-30\",\n      }",{"title":2376,"description":2377,"authors":2382,"heroImage":2378,"date":738,"body":2384,"category":718,"tags":2385},[2383],"David O'Regan","___Generative AI marks a monumental shift in the software development industry, making it easier to develop, secure, and operate software. Our new blog series, written by our product and engineering teams, gives you an inside look at how we create, test, and deploy the AI features you need integrated throughout the enterprise. Get to know new capabilities within GitLab Duo and how they will help DevSecOps teams deliver better results for customers.___\n\nAs AI becomes increasingly integral to software development, ensuring the security and thorough testing of AI-generated code is paramount. This article is a step-by-step guide to combining [GitLab Duo](https://about.gitlab.com/gitlab-duo/), our suite of AI features to powering the DevSecOps workflow, and [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/) to secure and thoroughly test AI-generated code. You'll discover how to mitigate common risks, including how to automatically generate tests, test code, and deploy test reports – all to enhance the reliability of your AI-generated code.\n\n> Discover the future of AI-driven software development with our GitLab 17 virtual launch event. [Watch today!](https://about.gitlab.com/seventeen/)\n\n## Challenges in AI-generated code\n\nAI-generated code often faces issues such as:\n\n- Mismatched algorithms: Incorrect or suboptimal algorithms can be generated.\n- Dependency problems: AI may include dependencies that are outdated or incompatible.\n- Security vulnerabilities: AI might generate code with potential security flaws.\n\nAI-generated code often faces issues such as mismatched algorithms, dependency problems, and security vulnerabilities. A [recent study published by the Association of Computing Machinery](https://dl.acm.org/doi/pdf/10.1145/3613904.3642596) on ChatGPT’s responses to programming questions found that 52% of the answers contained incorrect information, and 77% were overly verbose. Despite these shortcomings, users preferred ChatGPT’s comprehensive and well-articulated answers 35% of the time, even overlooking misinformation 39% of the time. Addressing these challenges requires the use of advanced tools and frameworks.\n\n## GitLab’s approach to AI security and testing\n\nGitLab has a comprehensive content strategy focused on integrating security measures within the development workflow. By leveraging GitLab Duo for AI-powered code generation and GitLab Pages for embedding test reports, developers can ensure their AI-generated code is both secure and reliable.\n\nHere is a step-by-step guide to pair GitLab Duo and GitLab Pages to ensure secure and thoroughly tested AI-generated code by implementing a [Flask web server](https://flask.palletsprojects.com/en/3.0.x/).\n\n### 1. Create a new project on GitLab.com\n\n- Go to [GitLab.com](http://GitLab.com).\n- Click on the \"New project\" button.\n- Choose \"Create blank project\".\n- Enter a project name (e.g., AI_Code_Security).\n- Set the visibility level (Public, Internal, or Private).\n- Click \"Create project\".\n\n### 2. Enable GitLab Duo Code Suggestions\n\n- Navigate to your project.\n- Click on the \"Web IDE\" button to open the Web IDE.\n- Ensure that GitLab Duo features like Code Suggestions and Duo Chat are enabled. \n- Start coding in the [Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/). As you type, GitLab Duo will provide code suggestions to help you write code more efficiently.\n\n### 3. Create a Flask web server\n\nYou can create a Flask web server using the comment (highlighted in green) in the screenshot below.\n\n![DGD testing - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097192/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097192520.png)\n\n### 4. Generate tests with GitLab Duo\n\nUnit tests are essential for validating the functionality of the generated code. Use GitLab Duo’s `/tests` command to [generate testing suggestions directly in the Web IDE](https://docs.gitlab.com/ee/user/gitlab_duo_chat_examples.html#write-tests-in-the-ide). This command can be tailored with additional instructions to focus on specific aspects such as performance, regression, or using particular frameworks.\n\n#### Example usage in Web IDE:\n\n- Select the code for which you want to generate tests.\n- Use the command `/tests` followed by additional instructions if needed.\n\n![DGD testing - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097192/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097192521.png)\n\n### 5. Identify issues with AI-generated code using GitLab Duo Chat\n\nUse GitLab Duo Chat to review and refine AI-generated code. For instance, let's check our Flask web server code for security vulnerabilities:\n\n```unset\nPrompt: Review this code for potential security vulnerabilities and dependency issues.\n\n```\n\n![DGD testing - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097192/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097192523.png)\n\nGitLab Duo Chat can help identify the vulnerabilities in the above code.\n\n### 6. Generate test reports\nAfter running your tests, generate a test report that will be deployed using GitLab Pages.\n\n```unset\n\nPrompt: Write me a python script to generate a test report that will be deployed using \nGitLab Pages.\n\n```\n\n![DGD testing - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097192/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097192525.png)\n\nWhat's happening here?\n\n- The script ensures the test_reports directory exists.\n- It runs the `test_server.py` file using `subprocess.run()`, capturing the output.\n- The raw output is saved to `test_reports/test_output.txt`.\n- An HTML report is generated, embedding the test output within `\u003Cpre>` tags for readability, and saved as `test_reports/index.html`.\n\n### 7. Deploy the test report with GitLab Pages\n\nUse [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/) to host and display the test report. Here’s the configuration for our `.gitlab-ci.yml` file to deploy the test report.\n\n```python\n\nstages:\n  - test\n  - deploy\ntest_job:\n  stage: test\n  script:\n    - python generate_test_report.py\n  artifacts:\n    paths:\n      - test_reports/\npages:\n  stage: deploy\n  script:\n    - mv test_reports public\n  artifacts:\n    paths:\n      - public\n\n ```\n\nWith this setup, the `test_job` stage runs the Python script to generate the test report. The `pages` stage moves the `test_reports` directory to `public`, which GitLab Pages uses to serve the content.\n\n### 8. Embedding test reports in MR widgets\n\nEmbedding the [test reports in MR widgets](https://docs.gitlab.com/ee/ci/testing/unit_test_reports.html) provides immediate visibility into the testing outcomes, ensuring transparency and reliability. This can be done by including the test report as an artifact in your CI/CD pipeline configuration:\n\n```python\n\nstages:\n  - build\n  - test\n  - deploy\n\nbuild_job:\n  stage: build\n  script:\n    - echo \"Building the project...\"\n    - # Your build commands here\n\ntest_job:\n  stage: test\n  script:\n    - mkdir -p test-reports\n    - python test_server.py > test-reports/results.xml\n  artifacts:\n    when: always\n    reports:\n      junit: test-reports/results.xml\n    paths:\n      - test-reports/results.xml\n\npages:\n  stage: deploy\n  script:\n    - mkdir .public\n    - mv test-reports .public/\n  artifacts:\n    paths:\n      - .public\n\n```\nBy including the test report as an artifact and specifying it in the reports section, GitLab will automatically display the test results in the MR widgets, providing immediate visibility into the testing outcomes and enhancing transparency and reliability.\n\n### Case study: AI reliability with security policies and scanners\n\nImagine a scenario where an AI-generated code snippet introduces a dependency that has known vulnerabilities. By using GitLab Duo and its security policies, this dependency would be flagged during the code generation process. Let’s consider an example where a snippet was generated by AI:\n\n```python\n\nimport os\nfrom flask import Flask, request\n\napp = Flask(__name__)\n\n@app.route('/search')\ndef search():\n    query = request.args.get('query')\n    execute_os_command(query)\n    return 'You searched for: ' + query\n\ndef execute_os_command(command):\n    os.system(command)\n\nif __name__ == '__main__':\n    app.run()\n\n```\n\nIn this example, the search endpoint is vulnerable to OS command injection. By leveraging GitLab's Static Application Security Testing ([SAST](https://docs.gitlab.com/ee/user/application_security/sast/)) component, this vulnerability would be detected during the CI/CD pipeline.\n\n#### Integrate SAST scanning to detect vulnerabilities\n\nGitLab SAST automatically analyzes your code for security vulnerabilities. Here’s how it can be integrated into your `.gitlab-ci.yml` file to scan for issues:\n\n```python\n\nstages:\n  - build\n  - test\n  - sast\n  - deploy\n\nbuild_job:\n  stage: build\n  script:\n    - echo \"Building the project...\"\n    - # Your build commands here\n\ntest_job:\n  stage: test\n  script:\n    - python test_server.py > test-reports/results.xml\n  artifacts:\n    when: always\n    reports:\n      junit: test-reports/results.xml\n    paths:\n      - test-reports/results.xml\n\nsast_job:\n  stage: sast\n  script:\n    - echo \"Running SAST...\"\n  artifacts:\n    reports:\n      sast: gl-sast-report.json\n  only:\n    - branches\n\npages:\n  stage: deploy\n  script:\n    - mv test-reports public\n  artifacts:\n    paths:\n      - public\n\n```\n\nIn this configuration, the `sast_job` stage runs SAST to detect vulnerabilities in the code, producing a report (`gl-sast-report.json`) that will be included in the pipeline artifacts! By integrating security policies and robust testing frameworks, GitLab Duo helps customers ensure that their AI-generated code is both efficient and secure.\n\n## Get started today\nThe integration of AI in software development brings significant benefits but also introduces new challenges. By using tools like GitLab Duo and GitLab Pages, developers can ensure that their AI-generated code is secure, well-tested, and reliable. Explore these tools today and join the conversation on enhancing AI security and testing!\n\n> [Start a trial of GitLab Ultimate](https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/blog&glm_content=default-saas-trial) today to access GitLab Duo and GitLab Pages.\n\n## Read more of the \"Developing GitLab Duo\" series\n\n- [Developing GitLab Duo: How we validate and test AI models at scale](https://about.gitlab.com/blog/developing-gitlab-duo-how-we-validate-and-test-ai-models-at-scale/)\n- [Developing GitLab Duo: AI Impact analytics dashboard measures the ROI of AI](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/)\n- [Developing GitLab Duo: How we are dogfooding our AI features](https://about.gitlab.com/blog/developing-gitlab-duo-how-we-are-dogfooding-our-ai-features/)\n- [Developing GitLab Duo: Blending AI and Root Cause Analysis to fix CI/CD pipelines](https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd/)\n",[720,851,9,830],{"slug":2387,"featured":6,"template":700},"how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code","content:en-us:blog:how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code.yml","How Gitlab Duo Helps Secure And Thoroughly Test Ai Generated Code","en-us/blog/how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code.yml","en-us/blog/how-gitlab-duo-helps-secure-and-thoroughly-test-ai-generated-code",{"_path":2393,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2394,"content":2399,"config":2404,"_id":2406,"_type":13,"title":2407,"_source":15,"_file":2408,"_stem":2409,"_extension":18},"/en-us/blog/how-gitlab-supports-the-fedramp-authorization-journey",{"title":2395,"description":2396,"ogTitle":2395,"ogDescription":2396,"noIndex":6,"ogImage":1620,"ogUrl":2397,"ogSiteName":685,"ogType":686,"canonicalUrls":2397,"schema":2398},"How GitLab supports the FedRAMP authorization journey","This comprehensive guide dives into the FedRAMP certification process, explaining how GitLab offers guidance and best practices for configuration and compliance.","https://about.gitlab.com/blog/how-gitlab-supports-the-fedramp-authorization-journey","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How GitLab supports the FedRAMP authorization journey\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Christian Nnachi\"}],\n        \"datePublished\": \"2024-08-07\",\n      }",{"title":2395,"description":2396,"authors":2400,"heroImage":1620,"date":2401,"body":2402,"category":830,"tags":2403},[1363],"2024-08-07","The Federal Risk and Authorization Management Program (FedRAMP) is a U.S. government program that standardizes security assessment, authorization, and continuous monitoring for cloud products and services. Achieving FedRAMP authorization allows cloud service providers (CSPs) to offer their services to federal agencies, ensuring that these services meet stringent security and privacy requirements.\n\nIn this article, you'll learn how to GitLab can help guide you on your FedRAMP authorization journey, including:\n* the key steps of the FedRAMP certification process\n* highlights of GitLab’s role in supporting FedRAMP requirements\n* best practices for configuration and compliance\n\nBy leveraging GitLab’s features and adhering to recommended practices, organizations can streamline their path to FedRAMP authorization and ensure secure and compliant software development.\n\n## Key requirements and compliance levels\n\nFedRAMP categorizes security requirements into [three levels based on the impact of data](https://www.fedramp.gov/understanding-baselines-and-impact-levels/) being handled:\n\n* **Low:** Impact on operations, assets, or individuals is limited.\n* **Moderate:** Impact on operations, assets, or individuals is serious.\n* **High:** Impact on operations, assets, or individuals is severe or catastrophic.\n\n## Security and privacy controls from NIST 800-53\n\nFedRAMP's security controls are derived from the [National Institute of Standards and Technology (NIST) Special Publication 800-53](https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final). Key areas include:\n\n* **Vulnerability scanning and patching SLAs:** Regular scanning and timely patching of vulnerabilities.\n* **Secure software supply chain:** Ensuring that the software and its components are secure.\n* **Change management:** Restricting unauthorized software or system changes through merge request (MR) approval rules.\n\n## Importance of FedRAMP for organizations\n\nFor CSPs, achieving FedRAMP authorization is crucial for doing business with federal agencies. Authorized services are listed on the [FedRAMP Marketplace](https://marketplace.fedramp.gov/products), enhancing their visibility and credibility.\n\n## Steps to achieve FedRAMP certification\n\nThe FedRAMP process is evolving, and a [new roadmap](https://www.fedramp.gov/2024-03-28-a-new-roadmap-for-fedramp/) has been introduced. To stay up to date on the latest changes, [subscribe to General Service Administration (GSA) list](https://public.govdelivery.com/accounts/USGSA/subscriber/new).\n\n### Walkthrough of the certification process\n\n#### 1\\. **Preparation and readiness**\n\n* **Preparation**\n  * Understand FedRAMP requirements and prepare documentation.\n* **Readiness assessment**\n  * CSPs can pursue the optional FedRAMP Ready designation by working with an accredited Third-Party Assessment Organization (3PAO). The 3PAO conducts a readiness assessment and documents the CSP's capability to meet federal security requirements in the Readiness Assessment Report (RAR).\n* **Pre-authorization**\n  * CSPs formalize partnerships with an agency as outlined in the FedRAMP Marketplace: Designations for Cloud Service Providers.\n  * CSPs prepare for the authorization process by making necessary technical and procedural adjustments to meet federal security requirements and prepare the required security deliverables for authorization.\n\n#### 2\\. **Authorization package submission and assessment**\n\n* **Authorization package submission**\n  * Historically: Submit the assessment package to the FedRAMP Joint Authorization Board (JAB) or a federal agency sponsor.\n  * [**New process**](https://www.fedramp.gov/2024-03-28-a-new-roadmap-for-fedramp/)**:** Submit to the FedRAMP Board within the GSA, replacing the JAB. The process integrates Agile principles and uses threat-based analysis for control selection and implementation.\n* **Full security assessment**\n  * The 3PAO conducts an independent audit of the CSP's system. Before this, the CSP should complete the System Security Plan (SSP) and have it reviewed and approved by the agency customer.\n  * The 3PAO develops the Security Assessment Plan (SAP) with input from the authorizing agency. After testing, the 3PAO creates a Security Assessment Report (SAR) detailing their findings and providing a recommendation for FedRAMP Authorization.\n* **Agency authorization process**\n  * The agency reviews the security authorization package, including the SAR, and may require CSP remediation.\n  * The agency performs a risk analysis, accepts the risk, and issues an Authority to Operate based on its risk tolerance, with the option to implement, document, and test customer-responsible controls either before or after the ATO issuance.\n\n#### 3\\. **Post-authorization and continuous monitoring**\n\n* **Continuous monitoring**\n  * The continuous monitoring phase involves post-authorization activities to maintain FedRAMP-compliant security authorization.\n* **New tool**\n  * [**automate.fedramp.gov**](https://www.fedramp.gov/2024-07-11-new-website-launch-automate-fedramp-gov/)**:** Provides detailed technical documentation, best practices, and guidance for creating and managing digital authorization packages with Open Security Controls Assessment Language ([OSCAL](https://pages.nist.gov/OSCAL/)). It supports a digital-first approach, offering faster documentation updates, enhanced user experience, and community collaboration.\n\nDetailed steps are available on the [FedRAMP Agency Authorization page](https://www.fedramp.gov/agency-authorization/). \n\n### Common challenges and pitfalls\n\n1. **Vulnerability management:** Ensuring timely and effective vulnerability management.\n2. **System boundaries:** Clearly defining and documenting system boundaries.\n3. **Software security practices:** Implementing and maintaining robust software security practices.\n4. **FIPS 140-2 cryptography:** Ensuring cryptographic modules are FIPS 140-2 compliant (details available in [GitLab's FIPS Compliance documentation](https://docs.gitlab.com/ee/development/fips_compliance.html)).\n\n## Role of self-managed GitLab in FedRAMP compliance\n\n### Supporting FedRAMP requirements\n\nSelf-managed GitLab can play a critical role in achieving FedRAMP compliance by providing tools and features that support secure code development and deployment within FedRAMP authorization boundaries.\n\n### Specific features of GitLab aligned with FedRAMP standards\n\n1\\. **Security configuration**\n\nYou can configure [CI/CD pipelines](https://docs.gitlab.com/ee/topics/build_your_application.html) to continuously test code while it ships and simultaneously enforce security policies. GitLab includes a suite of security tools that you can incorporate into the development of customer applications, including but not limited to:\n\n* [Security configuration](https://docs.gitlab.com/ee/user/application_security/configuration/index.html)\n* [Container scanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html)\n* [Dependency scanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html)\n* [Static application security testing](https://docs.gitlab.com/ee/user/application_security/sast/index.html)\n* [Infrastructure as code (IaC) scanning](https://docs.gitlab.com/ee/user/application_security/iac_scanning/index.html)\n* [Secret detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/index.html)\n* [Dynamic application security testing (DAST)](https://docs.gitlab.com/ee/user/application_security/dast/index.html)\n* [API fuzzing](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/index.html)\n* [Coverage-guided fuzz testing](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/index.html)\n\n2\\. **Access control and authentication**\n\nAccess management in a GitLab deployment varies for each customer. GitLab offers extensive documentation on deployments using both identity providers and GitLab's native authentication configurations. It is crucial to evaluate your organization's specific requirements before deciding on an authentication approach for your GitLab instance.\n\n3\\. **[Identity providers](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html#identity-providers)**\n\nTo comply with FedRAMP requirements, ensure your existing identity provider is FedRAMP-authorized and listed on the FedRAMP Marketplace, and for requirements like personal identity verification (PIV), use an identity provider rather than relying on native authentication in self-managed GitLab.\n\n4\\. **[Native GitLab user authentication configurations](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html#native-gitlab-user-authentication-configurations)**\n\nGitLab enables administrators to monitor users with different levels of sensitivity and access requirements.\n\n5\\. [**Audits and accountability**](https://docs.gitlab.com/ee/administration/audit_event_streaming/)\n\nGitLab provides a wide array of security events and streaming capabilities for comprehensive logging and monitoring that can be routed to a Security Information and Event Management (SIEM) solution.\n\n* [Event types](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html#event-types)\n\n6\\. **Incident response**\n\nAfter configuring audit events, it's crucial to monitor them. GitLab offers [tools](https://docs.gitlab.com/ee/operations/incident_management/index.html) for alert management, incident tracking, and status reporting through a centralized interface, allowing you to compile system alerts from SIEM or other security tools, triage incidents, and keep stakeholders informed.\n\n* [alerts](https://docs.gitlab.com/ee/operations/incident_management/alerts.html)\n* [incidents](https://docs.gitlab.com/ee/operations/incident_management/incidents.html)\n* [on-call schedules](https://docs.gitlab.com/ee/operations/incident_management/oncall_schedules.html)\n* [status page](https://docs.gitlab.com/ee/operations/incident_management/status_page.html)\n\n7\\. **Configuration management**\n\nAt its core, GitLab meets [configuration management](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html#configuration-management-cm) needs with robust CI/CD pipelines, approval workflows, and change control, primarily using issues and MRs to manage changes.\n\n8\\. **Federal Information Processing Standard (FIPS) compliance**\n\nGitLab supports [FIPS compliance](https://docs.gitlab.com/ee/development/fips_compliance.html) by offering versions that use FIPS-validated cryptographic modules such as OpenSSL, BoringSSL, or other CMVP-validated modules. This ensures that cryptographic operations meet FIPS requirements, making it suitable for use in environments that require high levels of security compliance, such as those seeking FedRAMP authorization. Additionally, GitLab's documentation provides detailed instructions for installing and configuring FIPS-compliant deployments, including a hybrid approach using omnibus and cloud native components.\n\n9\\. [**NIST 800-53 R5 security and privacy controls management project template**](https://gitlab.com/gitlab-org/project-templates/nist_80053r5)\n\nThe project template helps track and manage compliance with NIST 800-53 R5 using GitLab issues, based on [NIST 800-53R5 specifications](https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final). It includes pre-configured issues, issue boards, and a notional example pipeline to run tests using OpenSCAP (OSCAP) and update issues with artifacts and labels, creating a controls management project within GitLab. This template centralizes compliance efforts, automates control testing, and facilitates a seamless workflow for both project teams and auditors.\n\n## Best practices for using GitLab in the FedRAMP process\n\n### Recommended configurations and setups\n\nTo align self-managed GitLab with NIST 800-53 controls and FedRAMP requirements, consider the following best practices:\n\n1. **Security hardening:** Follow GitLab’s [security hardening guidance](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html).\n2. **Access control:** Implement role-based access control (RBAC) and enforce [the principle of least privilege](https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab/).\n3. **CI/CD pipelines:** Configure pipelines to include security testing and approval stages.\n4. **Audit logging:** Enable comprehensive audit logging and integrate with a SIEM system.\n5. **Backup and recovery:** Establish robust backup and recovery processes.\n\n### NIST 800-53 compliance\n\nGitLab provides various compliance features to help automate critical controls and workflows. Administrators should work with customer solutions architects to configure GitLab instances to meet applicable [NIST 800-53 controls](https://docs.gitlab.com/ee/security/hardening_nist_800_53.html).\n\n## Start your FedRAMP compliance journey\n\nAchieving FedRAMP authorization is a complex but strategic process for CSPs looking to provide services to federal agencies. Self-managed GitLab offers a comprehensive suite of tools and features that can support this journey, ensuring secure and compliant software development and operations. By following best practices and leveraging GitLab’s capabilities, organizations can navigate the challenges of FedRAMP compliance and successfully achieve authorization.\n\n> Learn more about [GitLab's solutions for the public sector](https://about.gitlab.com/solutions/public-sector/).",[9,185,851],{"slug":2405,"featured":90,"template":700},"how-gitlab-supports-the-fedramp-authorization-journey","content:en-us:blog:how-gitlab-supports-the-fedramp-authorization-journey.yml","How Gitlab Supports The Fedramp Authorization Journey","en-us/blog/how-gitlab-supports-the-fedramp-authorization-journey.yml","en-us/blog/how-gitlab-supports-the-fedramp-authorization-journey",{"_path":2411,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2412,"content":2417,"config":2423,"_id":2425,"_type":13,"title":2426,"_source":15,"_file":2427,"_stem":2428,"_extension":18},"/en-us/blog/how-gitlabs-red-team-automates-c2-testing",{"title":2413,"description":2414,"ogTitle":2413,"ogDescription":2414,"noIndex":6,"ogImage":2197,"ogUrl":2415,"ogSiteName":685,"ogType":686,"canonicalUrls":2415,"schema":2416},"How GitLab's Red Team automates C2 testing ","Learn how to apply professional development practices to Red Teams using open source command and control tools.","https://about.gitlab.com/blog/how-gitlabs-red-team-automates-c2-testing","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How GitLab's Red Team automates C2 testing \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Josh Feehs\"}],\n        \"datePublished\": \"2023-11-28\",\n      }",{"title":2413,"description":2414,"authors":2418,"heroImage":2197,"date":2420,"body":2421,"category":830,"tags":2422},[2419],"Josh Feehs","2023-11-28","At GitLab, our [Red Team](https://handbook.gitlab.com/handbook/security/threat-management/red-team/) conducts security exercises that emulate real-world threats. By emulating real-world threats, we help assess and improve the effectiveness of the people, processes, and technologies used to keep our organization secure. To operate effectively, we must utilize professional development practices like the threat actors we emulate.\n\n[Threat actors](https://www.securonix.com/blog/threat-labs-security-advisory-new-starkvortex-attack-campaign-threat-actors-use-drone-manual-lures-to-deliver-merlinagent-payloads/) use open source command and control (C2) tools such as [Merlin](https://github.com/Ne0nd0g/merlin). While convenient, these tools have intentionally detectable features to discourage illegitimate use. Red Teams often need to customize and combine different open source options to evade detections in the environments they target.\n\nIn this blog, you'll learn how our team applies professional development practices to using open source C2 tools. We'll share how we implement continuous testing for the Mythic framework, our design philosophy, and a public project you can fork and use yourself.\n\nOur solution, available in [this public project](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/continuousmage), improves our Red Team operations in two ways. First, it contains a suite of **pytest** tests for the Mythic C2 framework. These validate functionality of both the Mythic server and multiple Mythic-compatible agents. Second, it leverages **GitLab CI/CD pipelines** to automatically run these tests after each code change. This enables iterative development and rapid validation of updates to Mythic or Mythic-compatible C2 agents.\n\n## Prerequisites\n\nCurrently, a few prerequisites fall outside the scope of test automation:\n\n- A Linux VM with Mythic, its Python requirements, and the HTTP profile installed. See the [Mythic installation guide](https://docs.mythic-c2.net/installation). We suggest binding Mythic's admin interface to localhost only.\n- A fork of [the ContinuousMage GitLab project](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/continuousmage) in GitLab.com or your own GitLab instance. You'll build on top of this to run your own automation. We highly suggest making this fork private, so you don't expose your test infrastructure or C2 code changes.\n- GitLab Runner installed on the VM (configured with the [shell executor](https://docs.gitlab.com/runner/executors/shell.html)) and registered with your GitLab instance. See the docs on [installing](https://docs.gitlab.com/runner/install/) and [registering](https://docs.gitlab.com/runner/register/) a runner or follow the instructions provided when configuring your pipeline later in this blog. You'll assign this runner to your project when we configure CI/CD.\n- Your forked project cloned onto your VM. This allows testing code changes (or new tests) before triggering the pipeline.\n\n## Project structure\n\nThe project contains three main portions that we will detail in this blog post:\n\n1. `pytest` test code for running integration tests for Mythic and Mythic-compatible C2 agents\n2. The source of those Mythic-compatible C2 agents, as git submodules\n3. The GitLab CI/CD pipeline configuration that ties it all together\n\n## Part 1: pytests\n\n[pytest](https://docs.pytest.org/en/7.4.x/) is a framework for writing tests in Python. We can leverage pytest to do integration testing of Mythic since it has its own [Python package](https://pypi.org/project/mythic/). The test suite goals are:\n\n1. Be simple and atomic.\n2. Provide adequate coverage to validate tool readiness.\n\nWe'll walk through a simple test verifying an agent can run the `ls` command, highlighting key code sections for customization.\n\n### Implementation\n\n#### pytest file\n\nWhen run on a directory, `pytest` automatically discovers tests in files prefixed with `test_` and test functions starting with `test_`. Our tests are asynchronous, needing the `pytest.mark.asyncio` decorator, because the Mythic APIs we are testing are asynchronous. If your machine is missing test dependencies, run `python3 -m pip install mythic pytest pytest-asyncio`.\n\nA test function skeleton is as follows:\n\n```python\n@pytest.mark.asyncio\nasync def test_agent_ls():\n    # Will do the test here\n    continue\n```\n\n#### The GlMythic class\n\nThe `GlMythic` class wraps Mythic APIs for ease of use in testing. Because its `init` function is async, a coroutine creates the object:\n\n```python\n@pytest.mark.asyncio\nasync def test_agent_ls():\n    glmythic = await gl_mythic.create_glmythic()\n```\n\nBy default, it connects to the Mythic DB using the `MYTHIC_ADMIN_PASSWORD` environment variable and is configured to test the agent specified via the `AGENT_TYPE` environment variable. We will set these in the CI/CD config later.\n\n#### Interacting with Mythic via GlMythic\n\nWe'll include the remainder of the test code here, with comments, and then discuss the most important parts.\n\nAs a reminder, one of the key goals of this project was to make completely atomic tests. Each test only relies on a running Mythic server with the specific agent and HTTP containers loaded. As the test suite grows, it may be worth running a secondary set of tasks that relies on an already-existing agent connection. Currently, every test creates, downloads, and executes a new agent.\n\n### Test and deploy\n\n```python\n@pytest.mark.asyncio\nasync def test_agent_ls():\n\n    glmythic = await gl_mythic.create_glmythic()\n\n    # Unique payload_path per test\n    payload_path = \"/tmp/test_agent_ls\"\n\n    # Wraps agent create, download, and execute\n    proc = await glmythic.generate_and_run(payload_path=payload_path)\n\n    # Wait for callback\n    time.sleep(10)\n\n    # Uses the display_id field to determine most recent callback\n    # Assumes that the most recent callback is the one created by this test\n    callback = await glmythic.get_latest_callback()\n\n    # Issue the ls command, blocking on output\n    output = await mythic.issue_task_and_waitfor_task_output(\n        mythic=glmythic.mythic_instance,\n        command_name=\"ls\",\n        parameters=\"\",\n        callback_display_id=callback[\"display_id\"],\n        timeout=20,\n    )\n\n    # Clean up (no longer need the agent)\n    proc.terminate() \n    os.remove(payload_path)\n\n    # If the ls failed, there will be no output\n    # This test could also look for files in the repo (where the agent runs)\n    assert len(output) > 0\n\n```\n\nThe longest running portion of this test will be the call to `generate_and_run`, as agent builds within Mythic can take from seconds to minutes or even hang altogether. For your initial set of tests, sign in to the Mythic server and watch the **Payloads** screen for potential issues. In our testing, agent builds failed to complete around 5% of the time, depending on the agent. If you experience repeated build failures, reload your agent container with `sudo ./mythic-cli install folder \u003Cagent_directory> -f`.\n\nTo run the tests, run `pytest \u003Ctestfile_directory>`.\n\n## Part 2: Agent source as submodules\n\nBecause Mythic agents are often updated, we include the agent repos as git submodules in our test project. This allows us to update to new agent versions when they are released and use our project's version control to keep tool versions static for known good builds. These submodules are all located in the `agents` folder.\n\nWe'll discuss adding more agents to this project later in this blog.\n\n## Part 3: GitLab CI/CD pipeline\n\nNow that you have working pytests, you can automate your tests to run whenever you want. In our case, we chose to run our tests on merge requests and tagged commits (which are likely to be tool releases). We will be using [GitLab CI/CD pipelines](https://docs.gitlab.com/ee/ci/pipelines/) to perform our automated tests.\n\n### Configuring the pipeline\n\nNow is the time to set your GitLab CI/CD settings. To find these settings, go to your repository -> `Settings` -> `CI/CD`.\n\nThe first setting you'll want to set is your `Runner`. If you set up a runner as one of your prerequisite steps earlier, you can assign it here. If not, click `New project runner` and work through that process to create and set up your runner on your Mythic server. When you are prompted to choose a runner type on install, choose the [shell executor](https://docs.gitlab.com/runner/executors/shell.html). If your team uses shared runners for other CI/CD pipelines, you will want to make sure that shared runners are disabled for this project, given that your shared runners are unlikely to be able to talk to Mythic directly.\n\n![runner-settings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683075/Blog/Content%20Images/runner-settings.png)\n\nNext, you need to set your `Variables`. The `GlMythic` class uses the `MYTHIC_ADMIN_PASSWORD` environment variable to be able to actually sign into Mythic, so you need to make sure that the pipeline runner's environment is set up correctly.\n\nTo do this, click the `Add variable` button and add the `MYTHIC_ADMIN_PASSWORD` variable with the appropriate value. If you don't know your Mythic admin password, on the Mythic server in the directory where you installed Mythic, `cat .env | grep MYTHIC_ADMIN_PASSWORD` will give you the password.\n\nBecause GitLab handles merge requests in a detached state, you need to unclick the `Protect Variable` box, because that would prevent the pipeline from viewing the variable on a merge request otherwise. Because the variable is not protected, any branch committed back to your server can access your CI variables. This may pose a security risk if you allow remote access to your Mythic server (versus binding to localhost) and if you allow arbitrary users to access your repository. For this reason, our public repo does not have the environment variables. We use a private copy to perform testing, and suggest you do the same.\n\nAdditionally, set the `AGENT_TYPE` variable to the name of the agent you want to use. At time of release, valid agent types are `poseidon` or `merlin`. The section about adding more agents to the test suite will go into more detail.\n\nYou can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.\n\nNow that the pipeline is configured to use the runner and pick up the environment variables that you need, the only thing left to do is to set up your pipeline. This step is quite simple: If you add the `.gitlab-ci.yml` file to the root of your repository, GitLab will pick that up as the pipeline config on your next commit. Here is our example pipeline, which we will explain momentarily.\n\n```yaml\ninstall:\n  stage: install\n  script:\n    - sudo /opt/Mythic/mythic-cli install folder \"${CI_PROJECT_DIR}\"/agents/\"${AGENT_TYPE}\" -f\n  rules:\n    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'\n    - if: $CI_COMMIT_TAG\n\ntest:\n  stage: test\n  script:\n    - pytest \"${CI_PROJECT_DIR}\"/mythic-test\n  rules:\n    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'\n    - if: $CI_COMMIT_TAG\n```\n\nAll of the variables set above are made available by GitLab as part of every pipeline. This pipeline has two stages, `install` and `test`. Both stages are set to only run on merge requests or if the commit being evaluated has a specific tag. The `install` stage will install your C2 agent into Mythic using its local folder install. This makes sure that the Mythic server has your latest C2 code changes installed. Next, the `test` stage runs the set of pytest tests that we created. The `install` stage will run very quickly, and the `test` stage will run a little more slowly, given that it's doing the work of creating and interacting with Mythic agents.\n\n### Pipeline in action\n\nYou can do a couple of things to validate that your pipeline is working. First, if you are performing a merge request, there will be a section at the beginning of the merge request that will link to the pipeline. The screenshot below shows that the pipeline has passed, but you can click into the pipeline by clicking on its number even when it's running.\n\n![Pipeline passing](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683075/Blog/Content%20Images/merge-pipeline-pass.png)\n\nYou can then click into the stage that's running (or one that has already run) to view its output.\n\n![Pipeline task output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683075/Blog/Content%20Images/pipeline-task-output.png)\n\nAnd there you are! You now have working `pytest` tests for a Mythic agent that run every time you make a merge request.\n\n## Adapting for other agents\n\nWe tested our test suite against Poseidon and Merlin. Although the initial tests (generate, download and exec, ls) work the same for both agents, Poseidon and Merlin require different parameters for their `upload` commands. Unfortunately, this means that not all tests will be agent agnostic.\n\nAs a result, each `GlMythic` object that is created is told what type of agent it is testing. The coroutine for creating an object allows you to pass in the agent type as a variable, and defaults to using the `AGENT_TYPE` environment variable to determine which agent is being tested.\n\n```python\nasync def create_glmythic(  username=\"mythic_admin\",\n                            password=os.getenv(\"MYTHIC_ADMIN_PASSWORD\"),\n                            server_ip=\"127.0.0.1\",\n                            server_port=7443,\n                            agent_type=os.getenv(\"AGENT_TYPE\")):\n```\n\n### Agent source\n\nTo add more agents for testing, the first thing to do is to import your agent as a git submodule:\n\n```bash\ncd agents\ngit submodule add \"${URL_TO_YOUR_AGENT}\"\n```\n\nCommit your changes, and your agent is tracked as part of the repo.\n\n### Test compatibility\n\nYou'll need to validate that existing tests work with your agent. For tests to work, the parameters passed to the commands must match those in the test suite, with `upload` to be most likely to fail.\n\nThis is okay! Within the `test_agent_upload` test function, you'll see example code that specifies a different upload command for Merlin and Poseidon. Simply follow this structure for your own agent, passing your agent's parameters to the `mythic.issue_task_and_waitfor_task_output` function call.\n\nIf you are using another open source C2 and are unsure of the correct parameters to pass, you can use the Mythic UI. Interact with one of your agents and run the `upload` command to see what params you need to pass. If you do this for Poseidon, it will look like the following:\n\n![upload-parameters](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683075/Blog/Content%20Images/upload-parameters.png)\n\nOur test suite should be pretty easy to add to any Linux-based Mythic agent that supports the [HTTP C2 profile](https://github.com/MythicC2Profiles/http). Because the GitLab Runner installs the agent into Mythic (and Mythic is made to run on Linux), the runner is expecting to be on a Linux machine. Additional effort and test modifications will be required to run the test suite against a Windows or MacOS agent.\n\n## A quick win\n\nAs we worked on this project, we were continuously running our test suite against both Poseidon and Merlin. Unexpectedly, in early October 2023, our test for Poseidon's `upload` function started to fail. After a quick investigation, we identified that a bug had been introduced, present in Poseidon 2.0.2, that caused file uploads to fail.\n\nWe took our information to one of the Poseidon developers, Cody Thomas ([@its_a_feature_](https://twitter.com/its_a_feature_)), and he quickly identified the underlying issue and [fixed the problem](https://github.com/MythicAgents/poseidon/commit/83de4712448d7ed948b3e2d2b2f378d530b3a42a).\n\nThis highlights the usefulness of continuous testing. Instead of running into a potential bug during a Red Team exercise, we identified the issue beforehand and were able to report the bug so the issue was fixed.\n\nWe sincerely thank the Mythic, Merlin, and Poseidon developers for open sourcing their hard work. Many Red Teams around the world are able to perform high-quality security assessments in part because of the hard work of C2 developers who open source their tools. We also want to specifically thank Cody Thomas for addressing this bug within 20 minutes of notification. His responsiveness and attention to detail are unmatched.\n\n## Share your feedback\n\nThis post has demonstrated both the value of continuous testing and shown how to implement continuous testing for your own use, using GitLab. If you have worked alongside these examples, you've implemented some continuous testing for the Mythic framework and have tests that you can use for Merlin, Poseidon, or your own Mythic agent(s).\n\nAt GitLab, we always seek feedback on our work. If you have any questions or comments, please open an issue on [our project](https://gitlab.com/gitlab-com/gl-security/threatmanagement/redteam/redteam-public/continuousmage). You can also propose improvements via a merge request. We believe that everyone should be able to contribute, so we welcome any contributions, big or small.\n\n> [Try GitLab Ultimate for free today.](https://gitlab.com/-/trials/new)\n\n## Related reading\n- [Stealth operations: The evolution of GitLab's Red Team](https://about.gitlab.com/blog/stealth-operations-the-evolution-of-gitlabs-red-team/)\n- [How we run Red Team operations remotely](https://about.gitlab.com/blog/how-we-run-red-team-operations-remotely)\n- [Use GitLab and MITRE ATT&CK Navigator to visualize adversary techniques](https://about.gitlab.com/blog/gitlab-mitre-attack-navigator)\n- [Monitor web attack surface with GitLab](https://about.gitlab.com/blog/monitor-web-attack-surface-with-gitlab)\n",[830,1057,232,9],{"slug":2424,"featured":90,"template":700},"how-gitlabs-red-team-automates-c2-testing","content:en-us:blog:how-gitlabs-red-team-automates-c2-testing.yml","How Gitlabs Red Team Automates C2 Testing","en-us/blog/how-gitlabs-red-team-automates-c2-testing.yml","en-us/blog/how-gitlabs-red-team-automates-c2-testing",{"_path":2430,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2431,"content":2437,"config":2443,"_id":2445,"_type":13,"title":2446,"_source":15,"_file":2447,"_stem":2448,"_extension":18},"/en-us/blog/how-start-ospo-ten-minutes-using-gitlab",{"title":2432,"description":2433,"ogTitle":2432,"ogDescription":2433,"noIndex":6,"ogImage":2434,"ogUrl":2435,"ogSiteName":685,"ogType":686,"canonicalUrls":2435,"schema":2436},"Start an open source center of excellence in 10 minutes using GitLab","Launch your own open source program office using the OSPO Alliance's tools on GitLab","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682593/Blog/Hero%20Images/opensign.jpg","https://about.gitlab.com/blog/how-start-ospo-ten-minutes-using-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Start an open source center of excellence in 10 minutes using GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Boris Baldassari\"}],\n        \"datePublished\": \"2023-01-30\",\n      }",{"title":2432,"description":2433,"authors":2438,"heroImage":2434,"date":2440,"body":2441,"category":785,"tags":2442},[2439],"Boris Baldassari","2023-01-30","\nNow that open source has finally become a mainstream topic of conversation in the software industry, many organizations are increasingly curious about best practices for consuming, using, managing, and contributing to open source software projects. Open source software can seem alien and intimidating for organizations unfamiliar with it, and participating meaningfully and effectively in the open source ecosystem can be challenging.\n\nOrganizations especially serious about working in open source have formed [open source program offices](https://opensource.com/business/16/5/whats-open-source-program-office) (OSPOs) to spearhead their efforts. These offices are centers of excellence for an organization's ongoing work in open source. They help the organization realize the benefits of working with open source communities to accelerate innovation and build more secure tools.\n\nPerhaps your organization is considering establishing an OSPO. If it is, you likely have questions about how to get started – and especially about the best ways to help your organization become a valuable participant in the open source ecosystem.\n\nThe [OSPO Alliance](https://ospo.zone/) can help. Formed in 2021, the OSPO Alliance connects [experienced open source practitioners](https://ospo.zone/membership/) with organizations in need of seasoned guides to the open source world. Since the organization's founding, its members have composed a corpus of best open source practices called the [Good Governance Initiative Handbook](https://ospo.zone/ggi/), which explores various legal, cultural, and strategic considerations organizations face when working with open source software (and, naturally, the handbook itself is openly licensed, so anyone can contribute to it).\n\nTo celebrate the launch of the GGI Handbook Version 1.1, the OSPO Alliance went a step further: We have released the [MyGGI project](https://gitlab.ow2.org/ggi/my-ggi-board), which allows organizations to quickly create the infrastructure for their own open source program offices using GitLab.\n\nNow, let's look at what the MyGGI project can help your organization accomplish, including how to use the tool to establish an OSPO built on GGI principles — in only 10 minutes.\n\n## Working with the GGI Handbook\n\nThe GGI Handbook defines 25 activities, or best practices, organized according to various goals an organization may seek to accomplish with open source. Examples of activities include recommendations like \"Manage open source skills and resources,\" \"Manage software dependencies,\" \"Upstream first,\" or \"Engage with open source projects.\" Each of these activities, then, has a corresponding description and rationale, and the handbook provides resources, tools, and hints for successfully implementing them.\n\nActivites are intentionally generic and must be adapted to your organization's specific, unique, local context. The GGI Hanbook offers tools for doing this, too: scorecards. Scorecards allow you to assess your organization's engagement in and progress with various open source best practices.\n\nSo working with the GGI Handbook in your organization might look something like this:\n\n1. Evaluate the open source-related activities the handbook proposes and remove those that don't fit your specific context (maybe some activities will require a bit of adptation to be more relevant to the domain, while some others may just be discarded).\n1. Identify the activities that would be most beneficial to reaching your organization's goals in engaging with open source.\n1. Construct an Agile-like, iterative process for working on a small set of these activities. Do this in the form of sprints by tracking your progress with scorecards, and adapt the activity to your local context, team cultures, and available resources as you go.\n1. At the end of each iteration, review the activities your teams have completed, select a new scope for improvement, and repeat the process.\n\nThe MyGGI project provides a push-button infrastructure for doing this work. Next, let's examine how to deploy it on GitLab.\n\n### Deploying the GGI Handbook on GitLab\n\nThe OSPO Alliance wanted to provide a quick and straightforward way for organizations to establish their own open source program activities using a dashboard, so they can start implementing the GGI Handbook's methods without delay. We didn't want to reinvent the wheel with some heavy custom tooling. Instead, we decided to build the project using tools already available to us. We had already used GitLab issues to model activities during the early stages of handbook development, so reusing this GitLab feature made most sense. By simply adding some scripting to automate the initialization of activities and updating a static website on GitLab Pages, we were able to launch the project so others could easily deploy it in their own GitLab instances.\n\nInstructions for deploying the program are available in the project's [README](https://gitlab.ow2.org/ggi/my-ggi-board/-/blob/main/README.md). Let's review them here and start your own OSPO together.\n\nFirst, we need to create a new project on our GitLab instance. Select `Import project`, then `From repository by URL`. \n\nNext, we will need to provide a remote URL. Copy the existing MyGGI project by using the [URL](https://gitlab.com/gitlab-com/marketing/community-relations/open-source-program/gitlab-open-source-partners/publications-and-presentations/-/tree/main) `[https://gitlab.ow2.org/ggi/my-ggi-board.git](https://gitlab.ow2.org/ggi/my-ggi-board.git)`.\n\nThen we will give our project a unique name and choose a visibility level. Here's an example of how it might look:\n\n![Repository by URL](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-1.png){: .shadow}\n\nWhen you have configured your desired settings, click `Create project` to continue.\n\nOur next step is to configure access privileges. Go to `Project Settings > Access Tokens` and create a `Project Access Token` with `API` privilege and `Maintainer` role. The project's scripts will use these to create the issues and generate the static website dashboard for your OSPO.\n\nWhen the token is created, copy it to a safe place, as **you will never be able to see it again**. Note that some GitLab instances prefer to disable the Project Access Token feature in favor of Personal Access Tokens. This is perfectly okay; the preference won't affect the deployment of this project (see the instructions for more details).\n\nHere's an example of what you will see at this stage:\n\n![Project access tokens](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-2.png){: .shadow}\n\nWe will then provide this access token to the pipelines and scripts by creating a CI/CD environment variable. Go to `Project Settings` and then `CI/CD`. Scroll to the `Variables` section and add a new variable with name `GGI_GITLAB_TOKEN`. Input the access token you created in the last step as the value. Here's an example:\n\n![Add variable screen](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-3.png){: .shadow}\n\nWe can now execute the pipeline to begin the process of creating your OSPO infrastructure. Go to `CI/CD`, then `Pipelines`, and click on `Run pipeline`. After a couple of minutes, the pipeline should finish and the website will deploy. You will see something like this when the pipeline finishes:\n\n![Pipeline passed screen](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-4.png){: .shadow}\n\nInfrastructure for your open source program office is now ready!\n\n### Using the tools\n\nThe MyGGI project creates a set of 25 activities, along with a nice project board to help you visualize them:\n\n![Project board](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-5.png){: .shadow}\n\nUsers can click on specific activities (rendered as issues) to read the description of the activity, understand the tools and resources that might help them complete it, and begin completing relevant scorecards. Users can also define their own perspectives on the activities, as they see them from the organization's specific context. Then they can create tasks to narrow the scope of each activity so they can iterate on it and track progress. \n\nTheir work is displayed on a static website hosted on GitLab Pages and updated nightly according to the organization's progress on various activities and tasks. This web page is especially useful to present the program and its day-to-day evolution to the organization (or the world); participants, stakeholders, and executives can review it to learn more about the various initiatives, see what work is underway, and track the overall development of the organization's open source program office. The initial website looks like this:\n\n![Welcome screen of website](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-6.png){: .shadow}\n\n### Running your OSPO\n\nSelecting an open source program manager to oversee the work on the project boards is beneficial at this step. That person will:\n\n- Assign issues to team members to start working on new activities, create scorecards to track the work and associated tasks, and label them as \"In Progress\" instead of \"Not Started\".\n- Oversee the evolution of the work as it moves through various iterations, completing the scorecards with local resources and information, and closing issues as tasks are complete.\n- Ensure that issues keep making progress and, as team members complete them, assign new ones.\n\nAs changes occur in both the project and its issues, your OSPO's static website will regularly update to reflect the current status of activities, tasks, and the overall progress. After some time, for instance, the dashboard may look like this:\n\n![Dashboard with current status](https://about.gitlab.com/images/blogimages/ospointenminutes/screenshot-9.png){: .shadow}\n\nYou're now on your way to establishing your organization's open source program office. Don't hesitate to connect with the [OSPO Alliance](https://ospo.zone/) for help and support as you continue your journey!\n\n_Boris Baldassari is an open source consultant at the Eclipse Foundation Europe, and an active contributor to the OSPO Alliance._\n\nCover image by [Clay Banks](https://unsplash.com/@claybanks?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText) on [Unsplash](https://www.unsplash.com)\n{: .note}\n\n",[743,9,789],{"slug":2444,"featured":6,"template":700},"how-start-ospo-ten-minutes-using-gitlab","content:en-us:blog:how-start-ospo-ten-minutes-using-gitlab.yml","How Start Ospo Ten Minutes Using Gitlab","en-us/blog/how-start-ospo-ten-minutes-using-gitlab.yml","en-us/blog/how-start-ospo-ten-minutes-using-gitlab",{"_path":2450,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2451,"content":2457,"config":2462,"_id":2464,"_type":13,"title":2465,"_source":15,"_file":2466,"_stem":2467,"_extension":18},"/en-us/blog/how-to-access-gitlab-on-a-private-network-with-tailscale",{"title":2452,"description":2453,"ogTitle":2452,"ogDescription":2453,"noIndex":6,"ogImage":2454,"ogUrl":2455,"ogSiteName":685,"ogType":686,"canonicalUrls":2455,"schema":2456},"How to access GitLab on a private network with Tailscale","If issues around a private network were preventing a permanent GitLab installation, Brendan O'Leary has the solution with Tailscale.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679348/Blog/Hero%20Images/locks.jpg","https://about.gitlab.com/blog/how-to-access-gitlab-on-a-private-network-with-tailscale","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to access GitLab on a private network with Tailscale\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Brendan O'Leary\"}],\n        \"datePublished\": \"2022-07-21\",\n      }",{"title":2452,"description":2453,"authors":2458,"heroImage":2454,"date":2459,"body":2460,"category":785,"tags":2461},[1237],"2022-07-21","\nGitLab provides an easy-to-install package for most Linux distributions and even for devices like the [Raspberry Pi](https://docs.gitlab.com/omnibus/settings/rpi.html). However, if you want to install GitLab in a home lab or similar private network, you would then be faced with a new issue: how do you access the instance from outside that private network?\n\nTraditionally, you would set up your router to forward traffic from your public IP address to the server inside your network. However, this comes with several drawbacks:\n\n- Opening a port on your home or private network comes with a sustainable amount of risk.\n- It can be hard or impossible for folks to do depending on their internet service provider and what routing equipment they use.\n- It can be especially tough if your ISP doesn't provide you with a statically assigned IP address which means your address can change from time to time, and you'll need to either update DNS manually or through some third-party [dynamic DNS](https://www.cloudflare.com/learning/dns/glossary/dynamic-dns/) service.\n\nFor me, all of these challenges have meant that I've only ever really run GitLab \"for fun\" on my local network. Given the challenges above, running a permanent installation wasn't an option. That is until [Tailscale](https://tailscale.com) entered my life.\n\n## Tailscale\n\nTailscale isn't necessarily the \"newest\" technology. In fact, it is based on the [WireGuard protocol](https://www.wireguard.com/), which has existed in one form or another since 2015 and has seen native kernel support added to various Linux distributions as well as the kernel itself over the past several years.  Wireguard VPN technology makes considerable improvements in the usability and setup of virtual private networks over earlier protocols like IPsec. Even with being easier to use, the \"problem\" with WireGuard, at least for me, was always that it was still too complex to set up and maintain. Much like configuring my ISP's router for port forwarding, it wasn't _impossible_, but it just wasn't practical.\n\nEnter Tailscale. Tailscale provides a simple piece of client software, available for Linux, Mac, and Windows (and iOS and Android!), which implements the WireGuard protocol and allows you to control your VPN network from a handy web interface. Not only that, it's [free to use](https://tailscale.com/pricing/) for individuals and small networks. When I started using Tailscale, it was to make sure I could connect back to my home network and troubleshoot it while traveling for work. As the only system administrator in my house, this was fantastic.\n\nHowever, Tailscale also offers the ability to easily access services inside of various networks as well by setting up a mesh VPN between them, all with IP addresses in the 100.x.y.z range. That means for any web service or other service on my network, I can access it with a statically assigned IP address from any other device connected to Tailscale, and create a DNS record to have a domain point to the IP address. At last, I could run GitLab (and other open source tools) at home and safely connect to them from outside my house with as little hassle as possible. So how did I get it to work?\n\n## Tailscale and GitLab together\n\nAssuming you already have a GitLab [installation](/install/) up and running on your network, getting it working through Tailscale involves a few steps:\n\n- Installing Tailscale\n- Setting up DNS for the private address\n- Configuring HTTPS encryption\n\n### Installing Tailscale\n\nPackages are [available](https://tailscale.com/kb/1031/install-linux/) for many Linux distributions. To install Tailscale, you can select your [specific distribution](https://tailscale.com/kb/1031/install-linux/) for detailed instructions. There are also [static binaries](https://tailscale.com/kb/1053/install-static/) if you can't find your particular distribution - they are available for x86 and ARM CPUs for both 32- and 64-bit variants.\n\nOnce Tailscale is installed, getting it running is as simple as running the following command on the CLI:\n\n```bash\nsudo tailscale up\n```\n\nThe setup dialogue will walk you through the authentication process and get Tailscale running. After that process, you can see your new IP address for this node on your network with the CLI command `tailscale ip -4`. You'll need that IP address for the next steps.\n\nBy default, Tailscale will set an expiration date for the token it issues to your device during the authentication process. This is desirable for typical devices that may be transient or portable. However, suppose your device is secured inside your home or another secure place AND is a server you're not accessing all the time. In that case, you can optionally [disable key expiry](https://tailscale.com/kb/1028/key-expiry/) for that particular device.\n\n### Setting up DNS\n\nYou should be able to now access your device from any other Tailscale-connected device via the IP address from the last step. However, my goal was to make it easy for me to connect to GitLab, reference it by an URL, and encrypt the traffic end-to-end with TLS. As the next step I set up DNS.\n\nEven though the 100.x.y.z address is a private IP address, you can still create a public DNS record and have the hostname to point to it. That won't mean the whole world can access your server - it just means once you're connected to your Tailscale network, you can resolve that hostname to the IP address and access the web server. For me, I set up an A record for `gitpi.boleary.dev` to resolve to an IPv4 address:\n\n```\n;; QUESTION SECTION:\n;gitpi.boleary.dev.\t\tIN\tA\n\n;; ANSWER SECTION:\ngitpi.boleary.dev.\t300\tIN\tA\t100.64.205.40\n```\n\nAn important note here is that I use Cloudflare as my DNS provider - and I usually love Cloudflare's proxying service to make my \"real\" IP addresses private. In this case, you have to disable that proxying to make sure that you can resolve the correct address - Cloudflare can't proxy traffic into your Tailscale network.\n\n### Configuring HTTPS\n\nLastly, configuring HTTPS for your GitLab instance will ensure that all traffic is encrypted end-to-end. While Tailscale encrypts the traffic over the network, this will ensure there are no gaps between your device and your GitLab server.\n\nTo accomplish this, we'll use [`certbot`](https://certbot.eff.org/) from the EFF that lets us create and manage [Let's Encrypt](https://letsencrypt.org/) certificates. First, install `certbot` with `sudo apt install certbot` or follow the [instructions for your distribution](https://certbot.eff.org/instructions).\n\nAfter certbot is installed, issue a certificate to use with GitLab using a DNS challenge. Follow the steps to complete the DNS challenge after running this command:\n\n```bash\nsudo certbot certonly --manual --preferred-challenges dns\n```\n\nThe output will show you the specific location of the certificate it created (in my case, in a `gitpi.boleary.dev` folder), and you should link that certificate to GitLab's SSL directory by running:\n\n```bash\nsudo mkdir /etc/gitlab/ssl/\nsudo ln -s /etc/letsencrypt/live/gitpi.boleary.dev/fullchain.pem /etc/gitlab/ssl/gitpi.boleary.dev.crt\nsudo ln -s /etc/letsencrypt/live/gitpi.boleary.dev/privkey.pem /etc/gitlab/ssl/gitpi.boleary.dev.key\n```\n\nNext, configure GitLab to use the new certificate by opening the `gitlab.rb` with\n\n```bash\nsudo vi /etc/gitlab/gitlab.rb\n```\n\nAnd change the `external_url` value to match the URL for the certificate (e.g. `https://gitpi.boleary.dev`). That \"https\" will tell GitLab to enable TLS/SSL and use your linked certificate.\n\n## Finishing up\n\nThat's it! Now with a simple `gitlab-ctl reconfigure`, GitLab will pick up the new certificate and start responding to requests at that URL. From any device - iOS, Android, laptop, etc. - connected to your Tailscale network, you can access your GitLab installation (securely) from anywhere!\n",[9,743,232],{"slug":2463,"featured":6,"template":700},"how-to-access-gitlab-on-a-private-network-with-tailscale","content:en-us:blog:how-to-access-gitlab-on-a-private-network-with-tailscale.yml","How To Access Gitlab On A Private Network With Tailscale","en-us/blog/how-to-access-gitlab-on-a-private-network-with-tailscale.yml","en-us/blog/how-to-access-gitlab-on-a-private-network-with-tailscale",{"_path":2469,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2470,"content":2476,"config":2482,"_id":2484,"_type":13,"title":2485,"_source":15,"_file":2486,"_stem":2487,"_extension":18},"/en-us/blog/how-to-automate-creation-of-runners",{"title":2471,"description":2472,"ogTitle":2471,"ogDescription":2472,"noIndex":6,"ogImage":2473,"ogUrl":2474,"ogSiteName":685,"ogType":686,"canonicalUrls":2474,"schema":2475},"How to automate the creation of GitLab Runners","Follow this step-by-step guide for automating runner setup using new runner creation workflows.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664087/Blog/Hero%20Images/tanukicover.jpg","https://about.gitlab.com/blog/how-to-automate-creation-of-runners","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to automate the creation of GitLab Runners\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Darren Eastman\"}],\n        \"datePublished\": \"2023-07-06\",\n      }",{"title":2471,"description":2472,"authors":2477,"heroImage":2473,"date":2479,"body":2480,"category":785,"tags":2481},[2478],"Darren Eastman","2023-07-06","Automating the creation of GitLab Runners is an essential tactic in\noptimizing the operations and management of a runner fleet. Since announcing\nthe [deprecation and planned removal of the legacy runner registration\ntoken](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/#next-gitlab-runner-token-architecture)\nlast year, there have been various questions by customers and the user\ncommunity regarding the impact of the new workflow on any automation they\nrely on for creating and registering runners. This is a step-by-step guide\nfor automating runner setup using the new runner creation workflows as\ndepicted in the sequence diagram.\n\n\n![GitLab Runner create - sequence\ndiagram](https://about.gitlab.com/images/blogimages/2023-06-19-how-to-automate-creating-runners/runner_create_sequence_diagram.png){:\n.shadow}\n\n\n## New terminology and concepts\n\nBefore we dive into the automation steps, let’s first review a few new\nconcepts with the runner creation process and how that differs from the\nregistration token-based method. With the `registration token` method, a\n`registration token` is available for the instance, for each group, and for\neach project. Therefore, in a large GitLab installation, with many groups,\nsub-groups, and projects, you can have tens of hundreds of registration\ntokens that any authorized user can use to connect a runner. There are two\nsteps to authorizing a runner (the application that you install on a target\ncomputing platform) to a GitLab instance:\n\n1. Retrieve a registration token.\n\n2. Run the register command in the runner application using the previously\nretrieved registration token.\n\n\nThe workflow images below depict the runner setup steps using the\nregistration token compared with the new runner creation process.\n\n\n![GitLab Runner registration\nworkflows](https://about.gitlab.com/images/blogimages/2023-06-19-how-to-automate-creating-runners/runner_registration_workflows.png){:\n.shadow}\n\n\n### Reusable runner configurations\n\nNow, in the registration token method, if you authenticated multiple runners\nusing the same registration token (a valid use case), each runner entity\nwould be visible in the UI in a separate row in the list view. The new\ncreation method introduces the concept of a reusable runner configuration.\nFor example, if you have to deploy multiple runners at the instance level,\neach with the same configuration (executor type, tags, etc.), you simply\ncreate a runner and configuration **once**, then register each individual\nrunner with the same authentication token that you retrieved from the first\nrunner creation. Each of these runners is now displayed in the UI in a\nnested hierarchy.\n\n\n![Runner detailed view with shared\nconfigurations](https://about.gitlab.com/images/blogimages/2023-06-19-how-to-automate-creating-runners/runner_detail_shared_configs.png){:\n.shadow}\n\n\nWe heard from many of you that your Runners view was cluttered because each\nrunner created received its own row in the table, even if they were the\nexact same configuration as 100 others. With this change, our intent is to\nensure that you have the flexibility you need to configure a runner fleet at\nscale while ensuring that you can still easily understand and manage the\nfleet in the GitLab Runners view. We understand that this is a paradigm\nshift that may take some getting used to.\n\n\n## Automation steps for creating a runner\n\nHere are the automation steps to create a runner.\n\n\n### Step 1: Create an access token\n\nYou will first need to create an access token. A [personal access\ntoken](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)\nfor an administrator account will allow you to create runners at the\ninstance, group, and project levels.\n\n\nIf you only need to create a group or project runner, then it is best to use\na group access token or project access token, respectively. For a group or\nproject, navigate to `Settings / Access Tokens` and create a token. You must\nspecify a name, the token expiration date, role, and scope. For the role,\nselect `Owner`; for the scopes, select `create_runner`.\n\n\nNote: The access token is only visible once in the UI. You will need to\nstore this token in a secure location - for example, a secrets management\nsolution such as [Hashicorp\nVault](https://docs.gitlab.com/ee/ci/examples/authenticating-with-hashicorp-vault/)\nor the [Keeper Secrets Manager Terraform\nplugin](https://docs.keeper.io/secrets-manager/secrets-manager/integrations/terraform).\n\n\n![GitLab Runner registration\nworkflows](https://about.gitlab.com/images/blogimages/2023-06-19-how-to-automate-creating-runners/project_access_token.png){:\n.shadow}\n\n\n### Step 2: Use the access token to create a runner in the GitLab instance\n\nNow that you have an access token scoped to the instance, group, or project,\nthe next step is to use that token to create a runner automatically. In this\nexample, we will simply invoke a POST REST endpoint in a terminal using\nCURL.\n\n\n```\n\ncurl -sX POST https://gitlab.example.com/api/v4/user/runners --data\nrunner_type=group_type --data \"group_id=\u003Ctarget_group_or_project_id>\" --data\n\"description=software-eng-docker-builds-runner\" --data \"tag_list=\u003Cyour\ncomma-separated tags>\" --header \"PRIVATE-TOKEN: \u003Cyour_access_token>\"\n\n```\n\n\nOnce this step is complete, the newly created runner configuration is\nvisible in the GitLab UI. As the actual runner has not yet been configured,\nthe status displayed is `Never contacted`.\n\n\nThe API will return a message with the following fields: `id`, `token`, and\n`token_expires_at`. You must save the value for the `token` as it will only\nbe displayed once. \n\n\nAs mentioned above, a critical point to note in the new runner creation is\nthat you can reuse the runner token value to register multiple runners. If\nyou choose to do that, runners created with the same token will be grouped\nin the Runners list. Whichever runner contacted GitLab most recently will be\nthe one whose unique data (IP address, version, last contact time and\nstatus) displays in the list. You can still view all the runners in that\ngroup _and_ compare all of their unique data by going to the details page\nfor that runner. Each runner in the group is uniquely identified by their\n`system_id`.\n\n\nAt this point, you might ask yourself, what’s the difference between this\nnew workflow and the workflow that relies on the registration token? The\nbenefits are:\n\n1. You can now quickly identify the user that created a runner\nconfiguration. Not only does this add a layer of security compared to the\nold method, but it also simplifies troubleshooting runner performance\nissues, especially when your fleet expands.\n\n1. Only the creator of the runner or administrator(s) can edit crucial\nconfiguration details like tags, the ability to run untagged jobs, the\nsetting to lock to only run jobs in the current projects it is shared with,\nand more.\n\n\n## Automation of runner install and registration\n\nWith the runner configuration creation steps completed, you now have a\nrunner or runners configured in your GitLab instance and valid runner tokens\nthat you can use to register a runner. You can manually install the runner\napplication to a target compute host or automate the runner application\ninstallation. If you plan to host the runner on a public cloud virtual\nmachine instance – for example, [Google Cloud Compute\nEngine](https://cloud.google.com/compute/docs/instances) – then a good\n[example\npattern](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1932#note_1172713979)\nprovided by one of our customers for automating the runner install and\nregistration process is as follows:\n\n1. Use [Terraform infrastructure as\ncode](https://docs.gitlab.com/ee/user/infrastructure/iac/) to install the\nrunner application to a virtual machine hosted on GCP.\n\n1. Use the [GCP Terraform\nprovider](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance)\nand specifically the `metadata` key to automatically add the runner\nauthentication token to the runner configuration file on the newly created\nGCP virtual machine.\n\n1. Register the newly installed runner with the target GitLab instance using\na [cloud-init](https://cloudinit.readthedocs.io/en/latest/index.html#)\nscript populated from the GCP terraform provider.\n\n\n**Example cloud-init script**\n\n\n```shell\n\n#!/bin/bash\n\napt update\n\n\ncurl -L\n\"https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh\"\n| bash\n\nGL_NAME=$(curl 169.254.169.254/computeMetadata/v1/instance/name -H\n\"Metadata-Flavor:Google\")\n\nGL_EXECUTOR=$(curl\n169.254.169.254/computeMetadata/v1/instance/attributes/gl_executor -H\n\"Metadata-Flavor:Google\")\n\napt update\n\napt install -y gitlab-runner\n\ngitlab-runner register --non-interactive --name=\"$GL_NAME\"\n--url=\"https://gitlab.com\" --token=\"$RUNNER_TOKEN\"\n--request-concurrency=\"12\" --executor=\"$GL_EXECUTOR\"\n--docker-image=\"alpine:latest\"\n\nsystemctl restart gitlab-runner\n\n```\n\n\n## What's next?\n\nSo there you have it, an overview of how to automate runner creation,\ninstallation, and registration. To summarize in three simple steps:\n\n1. Use the API to create a runner token and configuration.\n\n1. Store the retrieved authentication token in a secrets management\nsolution.\n\n1. Use infrastructure as code to install the runner application on a target\ncompute host.\n\n\nOur long-term vision is to directly incorporate this automation lifecycle\ninto the product to simplify your day-to-day runner fleet management\noperations.\n",[9,956,744],{"slug":2483,"featured":6,"template":700},"how-to-automate-creation-of-runners","content:en-us:blog:how-to-automate-creation-of-runners.yml","How To Automate Creation Of Runners","en-us/blog/how-to-automate-creation-of-runners.yml","en-us/blog/how-to-automate-creation-of-runners",{"_path":2489,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2490,"content":2496,"config":2501,"_id":2503,"_type":13,"title":2504,"_source":15,"_file":2505,"_stem":2506,"_extension":18},"/en-us/blog/how-to-automate-localization-for-flutter-apps",{"title":2491,"description":2492,"ogTitle":2491,"ogDescription":2492,"noIndex":6,"ogImage":2493,"ogUrl":2494,"ogSiteName":685,"ogType":686,"canonicalUrls":2494,"schema":2495},"How to automate localization for Flutter apps","Follow this tutorial to learn how to simplify the localization process on GitLab with Localizely.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679465/Blog/Hero%20Images/flutterbanner.png","https://about.gitlab.com/blog/how-to-automate-localization-for-flutter-apps","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to automate localization for Flutter apps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2021-12-10\",\n      }",{"title":2491,"description":2492,"authors":2497,"heroImage":2493,"date":2498,"body":2499,"category":785,"tags":2500},[868],"2021-12-10","\n\nLocalization is an indispensable part of today's software. Almost all successful companies strive to adapt their products to different languages, regions, and cultures. Customer satisfaction is crucial for business. However, that often comes at a cost in terms of the higher complexity in software development and maintenance. In addition to regular activities, you must also take care of translation, its synchronization with development processes, and the like.\n\nThe question is: Can we somehow simplify the localization process and make it more agile? The answer is “yes.\" Below, you can see how GitLab and the [Localizely](https://localizely.com/) platform can help. For that purpose, we will use a simple Flutter project. However, the same approach can be applied to other programming languages and frameworks.\n\n## A few words about the Flutter project\n\nFlutter is an open-source framework developed by Google for building multi-platform apps from a single codebase. It has become quite popular lately, as it solves some things much better than some other solutions (hot-reload, performance, etc.). Since the point of this post is the automation of localization, we will not deal with Flutter too much. But we will certainly highlight some important things regarding localization in Flutter projects.\n\nWhatever approach you used to create and localize your Flutter project, its structure would probably be similar to the one below. \n\n![Flutter project structure](https://about.gitlab.com/images/blogimages/fluttergraphic.png){: .shadow.small.left}\n\nAbove, you can see the l10n folder with the two [ARB](https://localizely.com/flutter-arb/) files. Each ARB file contains translations for one language in the Flutter project (i.e. intl_de.arb for German and intl_en.arb for English). Whenever we want to add, modify, or remove a translation, we need to update those files. In other words, those files are the basis of localization in Flutter projects. They separate programming from translation but require synchronization with your code so that each message has a corresponding translation.\n\n## The usual way of localization\n\nThere is no exact rule or process that describes the usual way of localization. However, we could roughly describe it as the routine of a few steps:\n\n1. The developer updates code and the main ARB file.\n2. The developer sends ARB files to the project manager.\n3. The project manager sends ARB files to translators (e.g. email, upload to localization platform, etc.).\n4. The translators work on translations.\n5. The project manager forwards translated ARB files to the developer.\n6. The developer updates the Flutter project with new translations.\n\nIn this simplified case of localization, we can already notice some tasks that drain a lot of time and can be a bottleneck. Those are steps 2, 3, and 5. Moreover, these six steps can be frequent (e.g. update of the UI, new feature, etc.), which is not exactly the optimal solution. And that is even truer for medium and large teams. Just imagine how much time is wasted on file sharing when you have to coordinate in a team of 10+ people. Not to mention the problem with outdated ARB files.\n\n## Automated localization\n\nSince you've seen some flaws in the usual way of localization, let's see how we can optimize that.\n\n1. The developer updates code, the main ARB file, and pushes changes to GitLab.\n2. GitLab informs Localizely via webhook regarding new changes. \n3. Localizely fetches ARB files from GitLab and lets translators work on translations.\n4. The project manager pushes updated ARB files to GitLab via [MR](https://docs.gitlab.com/ee/user/project/merge_requests/).\n5. The developer updates the Flutter project with new translations (merge MR).\n\nThis way of working enables everyone to do their job more efficiently. Developers can be focused on the development of the product, translators on translations, managers on management, and similar. It should also be noted that with this type of workflow, you can easily accelerate the development and delivery of new features, which is in everyone's interest.\n\nTo make this workflow possible, you need to adjust a few things. In the following, you can see the necessary settings.\n\n1. Add a [localizely.yml](https://localizely.com/configuration-file/) config file to the root of your Flutter project. \n2. Set up [GitLab integration](https://localizely.com/gitlab-integration/) on the Localizely platform. \n3. Add a webhook to the GitLab repository.\n\nAnd that’s all. You have automated localization on your Flutter project. Whenever the developer pushes the changes to GitLab, the translators will see new string keys on the Localizely. Once the translation is done, a single click on the button creates a new MR with the latest translations on GitLab. There is no need for a mediator, waiting, or sending ARB files for every little thing. Now you can have more time for other things as this tedious work is automated.\n\n## Final thoughts\n\nIn this post, you have seen the most common steps of localization in Flutter projects and how to automate some of them. Knowing how important efficiency is today, we should strive to automate repetitive tasks as much as possible. As someone once said, “Lost time is never found again”.\n\n",[789,1367,9],{"slug":2502,"featured":6,"template":700},"how-to-automate-localization-for-flutter-apps","content:en-us:blog:how-to-automate-localization-for-flutter-apps.yml","How To Automate Localization For Flutter Apps","en-us/blog/how-to-automate-localization-for-flutter-apps.yml","en-us/blog/how-to-automate-localization-for-flutter-apps",{"_path":2508,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2509,"content":2515,"config":2521,"_id":2523,"_type":13,"title":2524,"_source":15,"_file":2525,"_stem":2526,"_extension":18},"/en-us/blog/how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci",{"title":2510,"description":2511,"ogTitle":2510,"ogDescription":2511,"noIndex":6,"ogImage":2512,"ogUrl":2513,"ogSiteName":685,"ogType":686,"canonicalUrls":2513,"schema":2514},"How to automatically create a new MR on GitLab with GitLab CI","With this script, every time we push a commit, GitLab CI checks if the branch that commit belongs to already has an open MR and, if not, creates one.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679783/Blog/Hero%20Images/whats-next-for-gitlab-ci.jpg","https://about.gitlab.com/blog/how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to automatically create a new MR on GitLab with GitLab CI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Riccardo Padovani\"}],\n        \"datePublished\": \"2017-09-05\",\n      }",{"title":2510,"description":2511,"authors":2516,"heroImage":2512,"date":2518,"body":2519,"category":785,"tags":2520},[2517],"Riccardo Padovani","2017-09-05","At [fleetster](https://www.fleetster.net/), we have our own instance of\n[GitLab](https://gitlab.com/) and we rely a lot on [GitLab\nCI](/solutions/continuous-integration/). How could it be otherwise? We are a\nsmall team, with a lot of different projects (only in last month, we had\nmore than **13,000 commits** over **25 different projects**, and we are only\n10 people – with myself working part time). Automating as many development\nsteps as possible (from build to QA to deploy) is helping us a lot, but\nsometimes we write some code and then forget about it. This is a disaster!\nWe have some bug fix or some new feature ready, but it is forgotten in some\nbranch somewhere.\n\n\n\u003C!-- more -->\n\n\nThis is why we have a policy to push as soon as possible to open a new MR,\nmark it as WIP, and assign to ourselves; in this way GitLab will remind us\nwe have an MR.\n\n\nYou need to do three steps to achieve that:\n\n\n* Push the code\n\n* Click on the link that appears on your terminal\n\n* Fill a form\n\n\nBut we are nerds. We are lazy. So one night, after a couple of beers,\n[Alberto Urbano](https://www.linkedin.com/in/alberto-urbano-047a4b19/) and I\nspent some hours to automate a task that requires 10 seconds.\n\n\nActually, the experience was quite fun, it was the first time we used GitLab\nAPIs and we learned things we will apply to others scripts as well.\n\n\n![Image via Riccardo's\nblog](https://about.gitlab.com/images/blogimages/automating-tasks-expectation-versus-reality.png){:\n.shadow}\u003Cbr>\n\n*Image by Randall Munroe,\n[xkcd.com](https://imgs.xkcd.com/comics/automation.png)*\n\n\n### The script\n\n\nWith this script, every time we push a commit, GitLab CI checks if the\nbranch that commit belongs to already has an open MR and, if not, it creates\nit. It then assigns the MR to you, and puts **WIP** in the title to mark it\nas a work in progress.\n\n\nIn this way you cannot forget about that branch, and when you’ve finished\nwriting code on it, you just need to remove the WIP from the title and\nassign to the right person to review it.\n\n\nIn the end, this is the script we came out with (when you add to your\nproject, remember to make it executable):\n\n\n```\n\n#!/usr/bin/env bash\n\n# Extract the host where the server is running, and add the URL to the APIs\n\n[[ $HOST =~ ^https?://[^/]+ ]] && HOST=\"${BASH_REMATCH[0]}/api/v4/projects/\"\n\n\n# Look which is the default branch\n\nTARGET_BRANCH=`curl --silent \"${HOST}${CI_PROJECT_ID}\" --header\n\"PRIVATE-TOKEN:${PRIVATE_TOKEN}\" | python3 -c \"import sys, json;\nprint(json.load(sys.stdin)['default_branch'])\"`;\n\n\n# The description of our new MR, we want to remove the branch after the MR\nhas\n\n# been closed\n\nBODY=\"{\n    \\\"id\\\": ${CI_PROJECT_ID},\n    \\\"source_branch\\\": \\\"${CI_COMMIT_REF_NAME}\\\",\n    \\\"target_branch\\\": \\\"${TARGET_BRANCH}\\\",\n    \\\"remove_source_branch\\\": true,\n    \\\"title\\\": \\\"WIP: ${CI_COMMIT_REF_NAME}\\\",\n    \\\"assignee_id\\\":\\\"${GITLAB_USER_ID}\\\"\n}\";\n\n\n# Require a list of all the merge request and take a look if there is\nalready\n\n# one with the same source branch\n\nLISTMR=`curl --silent \"${HOST}${CI_PROJECT_ID}/merge_requests?state=opened\"\n--header \"PRIVATE-TOKEN:${PRIVATE_TOKEN}\"`;\n\nCOUNTBRANCHES=`echo ${LISTMR} | grep -o\n\"\\\"source_branch\\\":\\\"${CI_COMMIT_REF_NAME}\\\"\" | wc -l`;\n\n\n# No MR found, let's create a new one\n\nif [ ${COUNTBRANCHES} -eq \"0\" ]; then\n    curl -X POST \"${HOST}${CI_PROJECT_ID}/merge_requests\" \\\n        --header \"PRIVATE-TOKEN:${PRIVATE_TOKEN}\" \\\n        --header \"Content-Type: application/json\" \\\n        --data \"${BODY}\";\n\n    echo \"Opened a new merge request: WIP: ${CI_COMMIT_REF_NAME} and assigned to you\";\n    exit;\nfi\n\n\necho \"No new merge request opened\";\n\n```\n\n\n### GitLab CI\n\n\nThe variables used in the script are passed to it by our `.gitlab_ci.yml`\nfile:\n\n\n```\n\nstages:\n    - openMr\n    - otherStages\n\nopenMr:\n    before_script: []   # We do not need any setup work, let's remove the global one (if any)\n    stage: openMr\n    only:\n      - /^feature\\/*/   # We have a very strict naming convention\n    script:\n        - HOST=${CI_PROJECT_URL} CI_PROJECT_ID=${CI_PROJECT_ID} CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} GITLAB_USER_ID=${GITLAB_USER_ID} PRIVATE_TOKEN=${PRIVATE_TOKEN} ./utils/autoMergeRequest.sh # The name of the script\n```\n\n\nAll these environment variables are set by GitLab itself, but the\nPRIVATE-TOKEN. A master of the project has to create it in its own profile\nand add to the project settings.\n\n\nTo create the personal token you can go to `/profile/personal_access_tokens`\non your GitLab instance, and then you add to your pipeline following this\nguide.\n\n\n### Ways to improve\n\n\nThe script is far from perfect.\n\n\nFirst of all, it has two API calls, one to take the list of MR and one to\ntake the default branch, to use it as target. Of course you can hardcode the\nvalue (in the end it shouldn’t change often), but hardcoding is always bad.\n\n\nAlso, it uses python3 to extract the name of the target branch – this is\njust one of many possible solutions, just use what is available on your\nsystem. Apart from that, the script doesn’t have any external dependency.\n\n\nThe other thing is how you need to set up the secret token to call the APIs.\nLuckily, GitLab’s developers are working on a [new\nway](https://gitlab.com/gitlab-org/gitlab-ce/issues/12729) to manage secret\ntokens.\n\n\n### Conclusion\n\n\nThis was a very small and very simple example about how much powerful\nContinuous Integration can be. It takes some time to set up everything, but\nin the long run it will save your team a lot of headache.\n\n\nIn fleetster we use it not only for running tests, but also for having\nautomatic versioning of the software and automatic deploys to testing\nenvironments. We are working to automate other jobs as well (building apps\nand publish them on the Play Store and so on).\n\n\nSpeaking of which, **do you want to work in a young and dynamic office with\nme and a lot of other amazing people?** Take a look at the [open positions\nat fleetster](https://www.fleetster.net/fleetster-team.html)!\n\n\nKudos to the GitLab team (and other guys who help in their free time) for\ntheir awesome work!\n\n\nIf you have any question or feedback about this blog post, please drop me an\nemail at riccardo@rpadovani.com :-)\n\n\nBye for now,\n\nA. & R.\n\n\nP.S: if you have found this article helpful and you’d like we write others,\ndo you mind to help us reaching the Ballmer’s peak and buy us a\n[beer](https://rpadovani.com/donations)?\n\n\nThis post originally appeared on\n[*rpadovani.com*](https://rpadovani.com/open-mr-gitlab-ci).\n\n\n## About the Guest Author\n\n\nRiccardo is a university student and a part-time developer at\n[fleetster](http://www.fleetster.net/). When not busy with university or\nwork, he likes to contribute to open-source projects.\n",[108,2288,9],{"slug":2522,"featured":6,"template":700},"how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci","content:en-us:blog:how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci.yml","How To Automatically Create A New Mr On Gitlab With Gitlab Ci","en-us/blog/how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci.yml","en-us/blog/how-to-automatically-create-a-new-mr-on-gitlab-with-gitlab-ci",{"_path":2528,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2529,"content":2535,"config":2541,"_id":2543,"_type":13,"title":2544,"_source":15,"_file":2545,"_stem":2546,"_extension":18},"/en-us/blog/how-to-choose-the-right-security-scanning-approach",{"title":2530,"description":2531,"ogTitle":2530,"ogDescription":2531,"noIndex":6,"ogImage":2532,"ogUrl":2533,"ogSiteName":685,"ogType":686,"canonicalUrls":2533,"schema":2534},"How to choose the right security scanning approach","GitLab offers multiple scanning methods for CI/CD pipelines, including compliance frameworks and scan and pipeline execution policies. Learn the basics, configurations, and advantages/disadvantages.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097969/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_282096522_securitycompliance.jpeg_1750097968823.jpg","https://about.gitlab.com/blog/how-to-choose-the-right-security-scanning-approach","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to choose the right security scanning approach\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Matt Genelin\"},{\"@type\":\"Person\",\"name\":\"Mathias Ewald\"}],\n        \"datePublished\": \"2024-08-26\",\n      }",{"title":2530,"description":2531,"authors":2536,"heroImage":2532,"date":2538,"body":2539,"category":830,"tags":2540},[2305,2537],"Mathias Ewald","2024-08-26","Integrating security scans into your CI/CD pipeline is crucial for\nmaintaining robust and secure applications. But who's responsible for those\nscans? Who is responsible for adding them into every CI/CD pipeline for all\nprojects? And who decides which identified vulnerability may pass or needs\nfixing? For organizations in regulated industries, these are critical\nquestions.\n\n\nIn this article, you'll learn how GitLab\n[CI/CD](https://about.gitlab.com/topics/ci-cd/) enables each person in the\nsoftware development lifecycle to incorporate security scanning. You'll also\ndiscover the advantages and disadvantages of the various options available\nto add scanning to GitLab project pipelines. Code examples will help you\nkickstart security scanning on the GitLab DevSecOps platform.\n\n\nArticle contents:\n\n- [The basics of setting up security\nscanning](#the-basics-of-setting-up-security-scanning)\n\n- [Pipeline includes](#pipeline-includes)\n\n- [Compliance frameworks](#compliance-frameworks)\n\n- [Policies](#policies)\n\n- [Get started with security scanning](#get-started-with-security-scanning)\n\n\n## The basics of setting up security scanning\n\n\nGitLab uses [fictional\npersonas](https://handbook.gitlab.com/handbook/product/personas/#user-personas)\nto describe the individual team member who would typically use a given\nsecurity feature or approach. By exploring the perspective of a **Software\nDeveloper (Sasha)**, **Application Security Engineer (Amy)**, or **Platform\nEngineer (Priyanka)**, you can better understand the needs of each role on\nyour team.\n\n\nGitLab follows a \"pipeline-per-project\" principle, stored in the file named\n`.gitlab-ci.yml`. This file contains the project's CI/CD pipeline definition\nand is revision controlled like any other file in the project. You'll learn\nabout these project pipelines, as well as compliance pipelines and policy\npipelines. While compliance pipelines and policy pipelines also refer to the\nYAML files in GitLab projects, they typically have a different file name and\nserve a different purpose.\n\n\nReaders already familiar with security scanning in GitLab will find clarity\nin the security pipeline choices available in the context of your\nteam/organization. Therefore, we will discuss each of the approaches with\nrespect to the following criteria:\n\n\n- **Ease of use:** How easy is it to add security scanning to project\npipelines? Is it a reasonable task for Sasha, or something that Amy and\nPriyanka should handle?\n\n\n- **Customization:** How deeply can scanner configurations be customized\nusing that approach? While default configurations that make sense and cover\na wide range of customer needs are worth gold, the time often comes when\nscanner configurations need adjustments.\n\n\n- **Enforcement:** Is this approach suitable to companies operating in\nregulated industries or that otherwise have global policies in place? Can we\nensure each relevant project runs Scanner X with Configuration Y?\n\n\n## Pipeline includes\n\n\n[GitLab project pipeline\nincludes](https://docs.gitlab.com/ee/ci/yaml/includes.html) are a mechanism\nthat allows the integration of external pipelines into the `.gitlab-ci.yaml`\nproject pipeline. This is similar to including a library in many programming\nlanguages. This powerful feature enables the seamless incorporation of your\nown templates, as well as GitLab-provided templates, to be used as building\nblocks for your pipelines. Includes can be used in project pipelines or\nother pipeline files. An example of a commonly included external pipeline is\nincluding a security scanning pipeline into a GitLab project pipeline.\n\n\nHere are the common types of includes, which use the security scanner\nexample.\n\n\n### Templates\n\n\nGitLab offers ready-to-use\n[templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Jobs)\nthat can be included in a project pipeline to make it easier for teams to\nadd in various pre-built elements. The following is example code:\n\n\n```yaml\n\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n  - template: Jobs/SAST.gitlab-ci.yml\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml\n  - template: Jobs/Container-Scanning.gitlab-ci.yml\n```\n\n\nThis code includes GitLab's templates for [Secret\nDetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/),\n[Static Application Security\nTesting](https://docs.gitlab.com/ee/user/application_security/sast/),\n[Dependency\nScanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/),\nand [Container\nScanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/)\n– all in only five lines of code. \n\n\nTo modify the behavior of jobs included via templates, you can either use\nvariables or use [GitLab's property merging\ncapabilities](https://docs.gitlab.com/ee/ci/yaml/includes.html#merge-method-for-include).\n\n\nYou will find an example of modifying the GitLab Container Scanning pipeline\nusing variables below. The [template for Container\nScanning](https://gitlab.com/gitlab-org/gitlab/-/blob/59f08760feaab1eb0489f694d4f28408af9c2e8d/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml)\nneeds to know the location of the image and uses a variable named `CS_IMAGE`\nfor that as is documented in the template code linked above.\n\n\n```yaml\n\nvariables:\n  CS_IMAGE: \"$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA\"\n\ninclude:\n  - template: Jobs/Container-Scanning.gitlab-ci.yml\n```\n\n\nThe project pipeline variables are available to included job templates by\ndefining the `CS_IMAGE` variable before the included pipeline template. The\nContainer Scanning template inherits the `CS_IMAGE` variable value. \n\n\nIf we wanted to make changes to the [`allow_failure` property defined\nhere](https://gitlab.com/gitlab-org/gitlab/-/blob/59f08760feaab1eb0489f694d4f28408af9c2e8d/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml#L38),\nwe would need to resort to property merging since the job templates employ\nno variable for the value. (The `allow_failure` property is a property\ngenerally available on every GitLab pipeline job. Please check the\n[documentation](https://docs.gitlab.com/ee/ci/yaml/#allow_failure) for\ndetails.)\n\n\nIn this example, `allow_failure` is set to `false`, meaning the entire\npipeline stops on a container scanning failure. This stops any unscanned\ncontainers from moving forward in the pipeline.\n\n\n```yaml\n\ninclude:\n  # Includes a job called \"container_scanning\"\n  - template: Jobs/Container-Scanning.gitlab-ci.yml\n\n# Define a job with same name for merging\n\ncontainer_scanning:\n  allow_failure: false\n```\n\n\nGitLab will load the job template and – as defined in the template code –\nregister a job called `container_scanning`. As the pipeline definition\ndeclares another job with that name, GitLab will merge that specification\nwith the already registered job.\n\n\nWhile this feature offers many possibilities, it also makes it impossible to\nprotect certain properties from being overwritten. We are only at the point\nof modifying the project pipeline, so there's no control over that anyway.\nBut later on, you will see that this can pose a challenge when security\nneeds to be enforced on a project.\n\n\n### Components\n\n\nTemplates are a great start for sharing repeatable GitLab pipelines. To\nfurther abstract reusable code across an entire organization or a GitLab\ninstance, [GitLab introduced\ncomponents](https://docs.gitlab.com/ee/ci/components/). Components are the\nnext logical step in GitLab's evolution of pipelines. Components are\ndesigned to simplify the creation and use of functional building blocks to\nuse in pipelines, or even to package and ship entire pipelines if needed.\nThey offer a well-defined interface, which accepts \"inputs\" for\nconfiguration. Otherwise, the component is completely isolated, which makes\nthem a great candidate to share work within an organization and to be\nsearchable and reusable building blocks.\n\n\nDevelopers can use the [CI/CD Catalog](https://gitlab.com/explore/catalog)\nto browse and search the collection of publicly available GitLab components,\nwhich are components officially built and maintained by GitLab. GitLab uses\nthe CI/CD Catalog [to publish our shipped\ncomponents](https://gitlab.com/components) such as security scanners\nalongside community-provided components.\n\n\nComponents are consumed similarly to templates via the `include` keyword. In\nan example above, we showed how the container scanning job requires\nknowledge of the image location. This \"input\" uses the component for\n[container\nscanning](https://gitlab.com/components/container-scanning/-/blob/19fd5b83bc631cb9890b4fadb08d31b3150853ce/templates/container-scanning.yml)\nis called `cs_image`. The configuration equivalent to the previous example\nlooks like this:\n\n\n```yaml\n\ninclude:\n  - component: $CI_SERVER_FQDN/components/sast/sast@2.0.2\n  - component: $CI_SERVER_FQDN/components/dependency-scanning/cargo@0.2.0\n  - component: $CI_SERVER_FQDN/components/secret-detection/secret-detection@1.1.2\n  - component: $CI_SERVER_FQDN/components/container-scanning/container-scanning@4.1.0\n    inputs:\n      cs_image: \"$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA\"\n```\n\nIn this example, the SAST component is pinned at Version 2.0.2, the\nDependency Scanning component at Version 0.2.0, the Secret Detection\ncomponent at Version 1.1.2, and the Container Scanning component at Version\n4.1.0. `~latest` [and more tags are\navailable](https://docs.gitlab.com/ee/ci/components/#component-versions) for\nbleeding-edge component usage and other development needs.\n\n\nWhether you use templates or components, your pipeline might look like the\nimage below. The top four jobs in the test stage are the result of the four\ninclude statements in the code above.\n\n\n![An example\npipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097984/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097983863.png)\n\n\n### Advantages and disadvantages of using pipeline includes\n\n\n#### Ease of use\n\n\nOne of the benefits of using pipeline includes in GitLab is their ease of\nuse. We have seen how, with essentially six lines of code, we included four\ncommonly used security scanners. All the complex logic and setup are handled\nwithin the templates or components, saving Sacha time and effort by\nproviding a ready-to-use solution.\n\n\n#### Customization\n\n\nWhile templates offer the highest flexibility (variables and merging), it's\nimportant to remember that with \"great power comes great responsibility.\"\nThe flexibility of templates supports extensive customization, but requires\ncareful management and oversight to avoid unexpected results.\n\n\nIn contrast, components provide a more structured mechanism for authoring,\nsharing, and maintaining building blocks for a broader audience. Components,\nwhile not as customizable, enhance stability and reliability, and are a\nvaluable, reusable, and repeatable feature. \n\n\n#### Enforcement\n\n\nAs the name _include_ suggests, it is the GitLab project pipeline that needs\nto include templates or components. While scanner templates are\nstraightforward to use, Amy and Priyanka cannot be sure Sacha has included\nthem properly, or even at all. Enforcement of scanner usage is needed.\n\n\nFor regulated industries, managing security in project pipelines is not an\napproach that provides the necessary audit trail or enforcement.\n\n\n## Compliance frameworks\n\n\nGitLab identified the gap between the ability to enforce security scans on\nproject pipelines and the need to [adhere to regulatory compliance\nframeworks](https://about.gitlab.com/blog/meet-regulatory-standards-with-gitlab/)\nsuch as PCI DSS, NIST, and many more. The introduction of compliance\nframeworks as functionality caters to precisely this challenge.\n\n\nAt first glance, a compliance framework in GitLab is merely a label attached\nto a project, which would typically be named after the regulatory framework\nit is supposed to implement. The magic is added with the link between that\nlabel and a compliance pipeline YAML file, which is responsible for\nimplementing the necessary steps to ensure compliance. \n\n\nThe mechanism is straightforward: Every time the project pipeline is\ntriggered, GitLab executes the compliance pipeline instead. The compliance\npipeline runs with both the [CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/) and [predefined CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)\nof the project pipeline.\n\n\nThis allows for two main design patterns: a \"wrapping pipeline,\" where the\ncompliance pipeline includes the project pipeline, and an \"overriding\npipeline,\" where it does not. \n\n\n**Note:** Compliance pipelines have been deprecated in GitLab Version 17.3\nand are scheduled for removal in Version 19.0. At this point, we cannot\nrecommend implementing this approach for new development platforms. However,\nyou might already be using them, making it worth reading this section.\n\n\n### Wrapping pipelines\n\n\nIn the wrapping approach, the compliance pipeline defines its own jobs\naccording to specific compliance needs. It includes the project pipeline in\nthe same way we have seen templates included in the previous section. This\nsetup is possible because the predefined CI/CD variables originate from the\nproject pipeline, allowing the system to identify the pipeline definition's\nlocation for inclusion.\n\n\nHere is an example of what a simple compliance pipeline might look like. \n\n\n```yaml\n\ninclude:\n  - component: $CI_SERVER_FQDN/components/sast/sast@2.0.2\n  - component: $CI_SERVER_FQDN/components/dependency-scanning/cargo@0.2.0\n  - component: $CI_SERVER_FQDN/components/secret-detection/secret-detection@1.1.2\n  - component: $CI_SERVER_FQDN/components/container-scanning/container-scanning@4.1.0\n  - project: '$CI_PROJECT_PATH'\n    file: '$CI_CONFIG_PATH'\n    ref: '$CI_COMMIT_SHA'\n```\n\n\nThe last three lines include the project pipeline based on available\nvariables.\n\n\n### Overriding pipelines\n\n\nUnlike wrapping pipelines, which include the project pipeline, overriding\npipelines ignore it entirely and run only their own jobs. This type of\npipeline defines each step, encompassing all necessary jobs to build, test,\nand deploy the application.\n\n\nBelow we see a mock compliance pipeline that illustrates this approach.\n\n\n```yaml\n\nstages: [\"build\", \"test\", \"deploy\"]\n\n\ninclude:\n  - component: $CI_SERVER_FQDN/components/sast/sast@2.0.2\n  - component: $CI_SERVER_FQDN/components/dependency-scanning/cargo@0.2.0\n  - component: $CI_SERVER_FQDN/components/secret-detection/secret-detection@1.1.2\n  - component: $CI_SERVER_FQDN/components/container-scanning/container-scanning@4.1.0\n\nbuild-job:\n  stage: build\n  script: echo \"Building the container image\"\n\ntest-job:\n  stage: test\n  script: echo \"Running unit tests\"\n\ndeploy-job:\n  stage: deploy\n  script: echo \"Deploying app\"\n```\n\n\n### Advantages and disadvantages of compliance frameworks\n\n\n#### Ease of use\n\n\nWhile compliance frameworks aren't terribly complicated, they aren't as\nstraightforward and simple as pipeline includes. They're meant to be written\nand assigned to projects by Amy and Priyanka, who now need to interact with\npipeline YAML code. A framework needs to be declared in the top-level\nnamespace and compliance pipelines need to be created and maintained, and\ncompliance frameworks need to be attached to the right projects. \n\n\n#### Customization\n\n\nAmy and Priyanka are the authors of compliance pipelines. Like Sacha in the\nprevious section on includes, they have full control over what they include\nand how they include it, giving them maximum customizability of compliance\njobs such as security scanners.\n\n\n#### Enforcement\n\nThis aspect of enforcing pipelines questions whether developers can tamper\nwith security jobs? In an environment with a strong separation of duties,\nthis nuance requires some extra attention. To answer this, we need to look\nat each pattern separately:\n\n\n##### Wrapping pipelines\n\nAs seen before, project pipelines are included in compliance pipelines. In\naddition to group- or project-level CI/CD variables, every element of that\nproject pipeline must be considered a potential threat to the compliance\npipeline. Obviously, variables and jobs stick out as primary candidates.\nAnd, in fact, they can and will influence security job behavior if used\nmaliciously.\n\n\nHere is a simple example to illustrate the issue.\n\n\nCompliance pipeline:\n\n```yaml\n\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n  - project: '$CI_PROJECT_PATH'\n    file: '$CI_CONFIG_PATH'\n    ref: '$CI_COMMIT_SHA'\n```\n\n\nProject pipeline:\n\n```yaml\n\nvariables:\n  SECRET_DETECTION_DISABLED: true\n\nsemgrep-sast:\n  rules:\n    - when: never\n```\n\n\nThis project pipeline declares a variable `SECRET_DETECTION_DISABLED` (this\ncould be done via project or croup-level CI/CD variables, too), which is\nevaluated in the included secret detection template. Further, the last three\nlines use the merging mechanism discussed previously, to not execute the job\nat all. Kind of redundant, we know.\n\n\nBoth overrides could be prevented using components, but you get the idea.\nComponents, too, are receptive to such attacks via their inputs' default\nvalues, which often use variables, too! Let's take a look at how this could\nbe taken advantage of.\n\n\nCompliance pipeline:\n\n```yaml\n\ninclude:\n  - component: $CI_SERVER_FQDN/components/sast/sast@2.0.2\n  - component: $CI_SERVER_FQDN/components/secret-detection/secret-detection@1.1.2\n  - project: '$CI_PROJECT_PATH'\n    file: '$CI_CONFIG_PATH'\n    ref: '$CI_COMMIT_SHA'\n```\n\n\nProject pipeline:\n\n```yaml\n\nvariables:\n  CI_TEMPLATE_REGISTRY_HOST: \"docker.io\"\n```\n\n\nTo understand what is happening here, look at the [SAST scanner component's\nLine\n6](https://gitlab.com/components/sast/-/blob/main/templates/sast.yml?ref_type=heads#L6):\n\n\n```yaml\n\nspec:\n  inputs:\n    stage:\n      default: test\n    image_prefix:\n      default: \"$CI_TEMPLATE_REGISTRY_HOST/security-products\"\n```\n\n\nThe `image_prefix` input uses the `CI_TEMPLATE_REGISTRY_HOST` to build the\ndefault value. By setting this variable to a false value in the same way we\nset `SECRET_DETECTION_DISABLED` to `true` before, Sacha may cause the job to\nload a wrong image and break SAST testing.\n\n\nTo prevent this override ability by the developer role, avoid templates in\nfavor of components. This approach covers many developer-induced loopholes.\nTo be certain of compliance, hardcode values for component inputs.\n\n\n##### Overriding pipelines\n\n\nThis type is an entirely different beast. Developers get no chance of\ninjecting actual pipeline code into the compliance pipeline. However,\ncompliance pipelines do run with the project's CI/CD variables. Hence, any\nvariable specified on the group- or project-level might modify the\ncompliance pipeline's behavior. With `SECRET_DETECTION_DISABLED` set to\n`true` in the project CI/CD variables, the following compliance pipeline can\nbe modified again:\n\n\n```yaml\n\nstages: [\"build\", \"test\", \"deploy\"]\n\n\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n\nbuild-job: ...\n\ntest-job: ...\n\ndeploy-job: ...\n\n```\n\n\nComponents can solve this particular problem, but, as before, component\ninputs may use CI/CD variables developers can set. Compliance pipeline\nauthors need to identify and take care of these situations. \n\n\n## Policies\n\n\nCompliance pipelines' shortcomings have led to the next step for managing\ncompliance:\n[policies](https://docs.gitlab.com/ee/user/application_security/policies/).\n\n\nGitLab introduced\n[policies](https://docs.gitlab.com/ee/user/application_security/policies/)\nas the way forward. Authors store a set of policies in a separate project as\nYAML files and apply them to projects on the group or project level. This\ngives Amy and Priyanka the flexibility to target individual projects with\nspecific requirements but also to ensure compliance across the entire\norganization if needed. Access to the policy project can be controlled\nwithin the policy project and audited within GitLab.\n\n\nPolicies come in different types for different purposes. The types we are\ninterested in right now are scan execution policies (SEP) and pipeline\nexecution policies (PEP).\n\n\n### Scan execution policies\n\n\nAs the name suggests, SEPs require a particular scan – or set of scans – to\nbe executed as part of the project pipeline and inject the respective scan\njobs into the pipelines of associated projects. They include the respective\n[template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Jobs)\nin the pipeline according to variables and rules set by Amy and Priyanka.\n\n\nGitLab supports policy authors with a comprehensive user interface in\naddition to a YAML-based Git workflow. The following screenshot and code\nsnippet illustrate a very basic example of a SEP:\n\n\n![Scan execution policy\nexample](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097984/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097983864.png)\n\n\n```yaml\n\nname: Secret Scanner\n\ndescription: ''\n\nenabled: true\n\nactions:\n\n- scan: secret_detection\n\nrules:\n\n- type: pipeline\n  branches:\n  - \"*\"\n```\n\n\nFor more details on SEP settings in the UI and YAML, please refer to the\n[policy\ndocumentation](https://docs.gitlab.com/ee/user/application_security/policies/scan_execution_policies.html).\n\n\n#### Advantages and disadvantages of scan execution policies\n\n\n##### Ease of use\n\nSEPs provide a lightweight, easy-to-use mechanism that enforces security on\nexisting and new CI/CD pipelines across the organization or on a granular\nlevel. The UI support makes them a viable tool for all relevant personas.\n\n\n##### Customization\n\nSEPs are restricted to predefined scanner jobs, and there is no option to\nextend this list with custom jobs at this point. This limitation can be\nrestrictive for teams with unique scanning requirements that fall outside\nthe standard options.\n\n\n##### Enforcement\n\n\nOnce an SEP is applied to a project (directly or indirectly), Sacha has no\nway to get rid of that scan job. Though, there may be ways to –\nintentionally or not – manipulate the scan job's behavior.\n\n\nJobs injected via SEPs generally are receptive to CI/CD variables and adhere\nto the general rules of [variable\nprecedence](https://docs.gitlab.com/ee/ci/variables/index.html#cicd-variable-precedence).\nFor this injection, Policies incorporate logic that denies changing some\npredefined variables as described\n[here](https://docs.gitlab.com/ee/user/application_security/policies/scan_execution_policies.html#cicd-variables)\nand generally deny the configuration of variables that follow certain\npatterns such as `_DISABLED` or  `_EXCLUDED_PATHS`.\n\n\nDespite these security measures, inconsiderate use of policies may still\nopen opportunities for tampering: In my test, I was able to set a\nproject-level CI/CD variable `SECURE_ANALYZERS_PREFIX` to a bad value (a\nnon-existing location) and as you can see\n[here](https://gitlab.com/gitlab-org/gitlab/-/blob/a2d4b8df0095c1363a105a1fa212daf227eca063/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml),\nthe secret detection template uses that to build the location of the scanner\nimage.\n\n\nWhile the scan job does get included in the pipeline run, it crashes very\nearly and, therefore, provides no scan results. Due to the [`allow_failure:\ntrue`\nconfiguration](https://gitlab.com/gitlab-org/gitlab/-/blob/a2d4b8df0095c1363a105a1fa212daf227eca063/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml#L18),\nthe pipeline will continue to run and eventually execute a deploy job.\n\n\nBecause SEP variables take the highest variable precedence, there is an easy\nfix to reduce the attack surface of the policy: Simply hardcode the correct\nvalue in your policy YAML or via the UI:\n\n\n```yaml\n\n- name: Secret Scanner\n  actions:\n  - scan: secret_detection\n    variables:\n      SECURE_ANALYZERS_PREFIX: registry.gitlab.com/security-products\n```\n\n\n### Pipeline execution policies\n\n\nSEPs enable the injection of a set of security-related jobs into any project\npipeline. In contrast, PEPs apply entire pipeline configurations to\nprojects, offering a lot more flexibility when it comes to customizing\nsecurity constraints. \n\n\nThere are two methods for implementing these policies, known as \"actions\":\n`inject` and `override`. These actions function similarly to the patterns we\nhave seen in the compliance frameworks section and provide flexible ways to\nenhance and enforce security standards within the development workflow.\n\n\n#### Injecting pipelines\n\n\nInjecting pipelines involves adding the jobs and other elements defined in\nthe policy pipeline into the project pipeline. Currently, jobs should only\nbe injected into reserved stages, namely `.pipeline-policy-pre` and\n`.pipeline-policy-post` to avoid unpredictable results.\n\n\nGitLab handles name clashes between jobs or variables in policy and project\npipelines effectively by building each pipeline in isolation before\ncombining them. This ensures that the integration process is seamless and\ndoes not disrupt existing workflows or configurations.\n\n\n![security scanning - image\n4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097984/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097983865.png)\n\n\nThe above screenshot shows an example of an injected policy pipeline.\nProject pipeline jobs are prefixed with `prj-` for easier identification.\n\n\n#### Overriding pipelines\n\n\nIn the override approach, the project pipeline is completely replaced by the\npolicy pipeline. This method is similar to compliance pipelines that do not\ninclude the project's `.gitlab-ci.yml` file. Despite the override, the\npipelines run using the project's CI/CD variables, maintaining consistency\nwith project-specific configurations. The compliance pipeline we used\nearlier makes a perfectly fine policy pipeline, too:\n\n\n```yaml\n\nstages: [\"build\", \"test\", \"deploy\"]\n\n\ninclude:\n  - component: $CI_SERVER_FQDN/components/sast/sast@2.0.2\n  - component: $CI_SERVER_FQDN/components/dependency-scanning/cargo@0.2.0\n  - component: $CI_SERVER_FQDN/components/secret-detection/secret-detection@1.1.2\n  - component: $CI_SERVER_FQDN/components/container-scanning/container-scanning@4.1.0\n\nbuild-job:\n  stage: build\n  script: echo \"Building the container image\"\n\ntest-job:\n  stage: test\n  script: echo \"Running unit tests\"\n\ndeploy-job:\n  stage: deploy\n  script: echo \"Deploying app\"\n```\n\n\nThe image below shows a slightly more complete pipeline than the mock\npipeline above:\n\n\n![More complete\npipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097984/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097983866.png)\n\n\n**Note:** This doesn't currently work with SEPs.\n\n\nHowever, the existence of a Dockerfile may not always be a valid indicator,\nas developers might be building without Dockerfiles using Cloud Native\nBuildpacks, Heroku Buildpacks, Kaniko, or other tools. Managed pipelines do\nnot encounter this challenge, as they are more controlled and centralized.\n\n\n\u003C!-- TOC ignore:true -->\n\n### Projects with multiple container images\n\nFor projects that produce multiple container images, several container\nscanning jobs would be necessary for proper coverage. This raises similar\nquestions as before: \"How do we know there are multiple?\" and \"Is the source\nof that information trustworthy?\". If we wanted to rely on the existence of\n`Dockerfile`s a [dynamic\napproach](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html#dynamic-child-pipelines)\nwould be necessary that includes a container scanning job for each\n`Dockerfile` detected.\n\n\n## Get started with security scanning\n\nIn this article, you've learned about a variety of approaches to adding\nsecurity scanning to CI/CD pipelines with a close look at ease of use,\ncustomizability, and the ability to strictly enforce scanning. You've seen\nthat a pipeline author who is held responsible for project compliance needs\nto keep a few things in mind during the process to avoid surprises down the\nline. We recommend building a small testing space on your GitLab instance\nand then run a few tests to reproduce the main points of this article. Put\nyourself in the shoes of a malicious Sacha (Sachas aren't generally\nmalicious people, but it's a good exercise) and think about how you could\nfool that annoying Amy and her security scans.\n\n\nGitLab provides strong support for all sorts of requirements and all\napproaches are – at least in our eyes – easy to implement due the platform's\nbaked-in functionality. You should find ways to bulletproof your scan jobs\nand, if not, you should open a ticket with our support. \n\n\nHappy pipelining!\n\n\n> #### Get started with security scanning today!\n\n> [Sign up for a free trial of GitLab\nUltimate](https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com/blog&glm_content=default-saas-trial)\nto implement security scanning in your software development lifecycle.\n\n\n## Read more\n\n\n- [Meet regulatory standards with GitLab security and\ncompliance](https://about.gitlab.com/blog/meet-regulatory-standards-with-gitlab/)\n\n- [How to integrate custom security scanners into\nGitLab](https://about.gitlab.com/blog/how-to-integrate-custom-security-scanners-into-gitlab/)\n\n- [Integrate external security scanners into your DevSecOps\nworkflow](https://about.gitlab.com/blog/integrate-external-security-scanners-into-your-devsecops-workflow/)\n",[830,9,108],{"slug":2542,"featured":90,"template":700},"how-to-choose-the-right-security-scanning-approach","content:en-us:blog:how-to-choose-the-right-security-scanning-approach.yml","How To Choose The Right Security Scanning Approach","en-us/blog/how-to-choose-the-right-security-scanning-approach.yml","en-us/blog/how-to-choose-the-right-security-scanning-approach",{"_path":2548,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2549,"content":2554,"config":2559,"_id":2561,"_type":13,"title":2562,"_source":15,"_file":2563,"_stem":2564,"_extension":18},"/en-us/blog/how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod",{"title":2550,"description":2551,"ogTitle":2550,"ogDescription":2551,"noIndex":6,"ogImage":2259,"ogUrl":2552,"ogSiteName":685,"ogType":686,"canonicalUrls":2552,"schema":2553},"How to code, build, and deploy from an iPad using GitLab and Gitpod","Senior Developer Evangelist Brendan O'Leary tackles the challenge of doing DevOps from a tablet.","https://about.gitlab.com/blog/how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to code, build, and deploy from an iPad using GitLab and Gitpod\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Brendan O'Leary\"}],\n        \"datePublished\": \"2022-02-10\",\n      }",{"title":2550,"description":2551,"authors":2555,"heroImage":2259,"date":2556,"body":2557,"category":785,"tags":2558},[1237],"2022-02-10","As a software engineer, it can be tough to go all-in on just using an iPad\nfor your daily driver. So when Apple announced the M1 chip-based iPads, I,\nalong with many techies, got excited to see if we'd finally get things like\na proper terminal on the iPad. But that still isn't the use case that the\niPad solves. I remained determined to be able to *code* from mine. So I\nhooked up my magic keyboard and fired up Gitpod to code and GitLab to build\nand deploy an app from scratch... all from my iPad.\n\n\n## Getting started\n\n\nLike any of [my projects](/blog/introducing-auto-breakfast-from-gitlab/),\nthe first thing I needed was inspiration. I had promised my colleague\n[Pj](https://brendan.fyi/pj) for some time that I would review [his\nblog](https://brendan.fyi/pj-twitter-blog) on how to make a Twitter bot like\nall of the fantastic ones he built while breaking into tech. Combine the\nneed to learn the Twitter API to provide an excellent review with my love of\nElton John's music, and I had it: I'd make a Twitter bot that tweeted every\nmorning at 4:00 am (as an homage to the line in “Someone Saved My Life\nTonight”).\n\n\nArmed with my newfound inspiration, I ran to gitlab.com in Safari (on my\niPad, obviously) and created a new, blank GitLab project.\n\n\n![ipad on\ndesk](https://about.gitlab.com/images/blogimages/brendanipad1.png){:\n.shadow}\n\n\n## Coding on the iPad\n\n\nOnce I had the new project, getting started on Gitpod was as easy as\nclicking the \"Gitpod\" button on GitLab to open my repository in Gitpod.\n\n\nGitpod enables you to access an entire development environment from any\nbrowser. By default, you get a container with many development tools (Node,\nRuby, OpenJDK, etc.). But you can also choose [your own\ncontainer](https://www.gitpod.io/docs/config-docker) as a starting point\nwith a .gitpod.yml… but we'll talk about that later.\n\n\nThe environment is presented to you as a VS Code interface – where you can\nopen, edit, and add files just as you'd expect. You can also access the\nterminal just like you would in VS Code and install anything you might need\nto get your project running.\n\n\nIn this example, I decided to build the Twitter bot in Node.js, so I\ninitialized a new Node project and installed the packages I'd need with:\n\n\n```bash\n\nnpm init -y\n\nnpm install express twit node-schedule dotenv\n\n```\n\n\n## Running your app\n\n\nOnce I had some code running – just the [Express sample\napp](https://expressjs.com/en/starter/hello-world.html) that says Hello\nWorld – running the app was just as easy as if I was going to run it on my\nlaptop:\n\n\n```bash\n\nnpm dev\n\n```\n\nNot only did that run my code to connect to the Twitter API, wait until 4:00\na.m. (UTC), and then tweet to let everyone know it was 4:00 a.m., but it\nalso shows this relative to my Express app:\n\n\n![Express\napp](https://about.gitlab.com/images/blogimages/brendanipad3.png){: .shadow}\n\n\nThat allows me to preview my [website for the\napp](https://brendan.fyi/4oclock) while I'm coding it. This is a massive\nbenefit because it means I can have two tabs open on the iPad – one with\nGitpod and my code and another with the website as I change it. Or I can\neven use split-screen on the iPad to have them side-by-side like I might if\nI was at my desk at my \"normal\" setup. And there's even a button to make the\nsite available publically so I could share it with my team and show them\nwhat I'm working on (as long as my Gitpod workspace is running).\n\n\nNow, when it comes to coding the rest of the Twitter bot, I used the\npreviously mentioned [tutorial](https://brendan.fyi/pj-twitter-blog) from my\ncolleague [Pj](https://brendan.fyi/pj). So I won't go into detail on the\nactual coding of the app – you can find the\n[code](https://gitlab.com/brendan-demo/4oclock),\n[website](https://brendan.fyi/4oclock), and [Twitter\nbot](https://twitter.com/DammitOclock) if you want to learn more about the\napp itself. But to deploy the website and the bot, I needed something else:\n[GitLab CI/CD](https://docs.gitlab.com/ee/ci/).\n\n\n## Deploying the app\n\n\nCombining GitLab CI/CD and GitLab.com's SaaS offering with Gitpod meant that\nI could not only code and preview the app from my iPad, but I could also get\nit deployed to Heroku (or any provider) from the couch. \n\n\nI created a `.gitlab-ci.yml` file in my project to get started. For\ndeploying to Heroku:\n\n\n- I like to use a Ruby package called\n[dpl](https://github.com/travis-ci/dpl) from Travis CI because it makes it a\nsimple one-line command.  Alternatively, I could install the [Heroku\nCLI](https://devcenter.heroku.com/articles/heroku-cli) and deploy with that\nif I wanted to. \n\n\n- I added the `HEROKU_API_KEY` variable to my [CI/CD\nvariables](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project)\nso that I could authenticate with Heroku for the deployment. \n\n\n- I then set the `rules:` section to only deploy when commits are impacting\nthe main (default) branch, and I was ready to go! \n\n\nNow, every time I push code from Gitpod to GitLab, GitLab will start the\nbuild and deploy it to Heroku:\n\n\n```yaml\n\nimage: starefossen/ruby-node:2-10\n\n\nvariables:\n APP_NAME: four-oclock-in-the-morning\n\ndeploy:\n stage: deploy\n script:\n - gem install dpl -v 1.10.6\n - dpl --provider=heroku --app=$APP_NAME --api-key=$HEROKU_API_KEY\n rules:\n - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH\n```\n\n\n## Enabling collaboration\n\n\nThere are two other concepts that this pattern introduces that are worth\ndiscussion: the idea of one environment per change and enabling new\ncollaborators to spin up a development environment in seconds.\n\n\nMost developers are used to having our setup just the way we like it –\nprecisely the correct number of monitors, keys on our keyboard, and all of\nour favorite tools installed. However, that can lead to issues. We already\nknow we should treat our servers like cattle, not pets, so why do we still\ntreat our laptops like pets? While I love my MacBook and the stickers on it\nas much as the next person, I can get frustrated when setting up a new one\nand getting it back to the way I like it.\n\n\nIn addition, on many projects I've been on in the past, onboarding a new\ndeveloper can take a lot of effort, including getting the correct libraries\ninstalled and ensuring they have access to all the right resources and\nenvironments. These things may seem trivial, but I've seen it take up to\nthree days from senior engineers just to get another engineer up and\nrunning. All of that time is time that could be much better spent on writing\ncode for the actual business.\n\n\nGitpod solves both of these issues with a simple YAML file: `.gitpod.yml`.\nThis file allows you to specify:\n\n\n- What image to use as the base for the environment\n\n- Which other tools to install\n\n- What commands to run at startup, and even things like which VSCode\nextensions you should have in the environment\n\n\nAnd [lots of different\nsettings](https://www.gitpod.io/docs/references/gitpod-yml) that you can\nfind in the [Gitpod docs](https://www.gitpod.io/docs).\n\n\nSpecifying all of the tools needed lets you have short-lived environments\nthat you can spin up for one task and then discard and get a fresh one for\nthe next task. And it also saves time when onboarding new engineers by\nguaranteeing they have a running system within just a few seconds of opening\nthe project. Best of all, it is all in a file that's in source control, so\nas things change or you make improvements to the development environment,\nall of your developers benefit from it immediately.  \n\n\nI added a simple\n[`.gitpod.yml`](https://gitlab.com/brendan-demo/4oclock/-/blob/main/.gitpod.yml)\nto run `npm run dev` to get started when you create a new environment. That\nsimple example is great for a simple Node app or similar, but what about\nsomething more complex? Gitpod works for that, too. GitLab itself has a\n[`gitpod.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitpod.yml)\nthat lets you get an entire working GitLab development environment – and all\nthat entails – up and running quickly, without the need to install Postgres\nand Redis and all of the other dependencies GitLab has.\n\n\nThis makes contributing to GitLab easier than ever. Just go to the [GitLab\nrepository](https://brendan.fyi/gitlab-repo) and click on that Gitpod button\nto get started. I'd love to hear how it works for you!\n",[789,9,765],{"slug":2560,"featured":6,"template":700},"how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod","content:en-us:blog:how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod.yml","How To Code Build And Deploy From An Ipad Using Gitlab And Gitpod","en-us/blog/how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod.yml","en-us/blog/how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod",{"_path":2566,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2567,"content":2573,"config":2580,"_id":2582,"_type":13,"title":2583,"_source":15,"_file":2584,"_stem":2585,"_extension":18},"/en-us/blog/how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab",{"title":2568,"description":2569,"ogTitle":2568,"ogDescription":2569,"noIndex":6,"ogImage":2570,"ogUrl":2571,"ogSiteName":685,"ogType":686,"canonicalUrls":2571,"schema":2572},"CI/CD pipeline: GitLab & Helm for Kubernetes Auto Deploy","One user walks through how he tried GitLab caching and split the job into multiple steps to get better feedback.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664472/Blog/Hero%20Images/gitlabflatlogomap.png","https://about.gitlab.com/blog/how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to create a CI/CD pipeline with Auto Deploy to Kubernetes using GitLab and Helm\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sergey Nuzhdin\"}],\n        \"datePublished\": \"2017-09-21\",\n      }",{"title":2574,"description":2569,"authors":2575,"heroImage":2570,"date":2577,"body":2578,"category":785,"tags":2579},"How to create a CI/CD pipeline with Auto Deploy to Kubernetes using GitLab and Helm",[2576],"Sergey Nuzhdin","2017-09-21","Recently, I started working on a few Golang\n[microservices](/topics/microservices/). I decided to try GitLab’s caching\nand split the job into multiple steps for better feedback in the UI.\n\n\n\u003C!-- more -->\n\n\nSince my previous\nposts[[1](http://blog.lwolf.org/post/how-to-build-tiny-golang-docker-images-with-gitlab-ci/)][[2](http://blog.lwolf.org/post/continuous-deployment-to-kubernetes-from-gitlab-ci/)]\nabout [CI/CD](/topics/ci-cd/), a lot has changed. I started using Helm\ncharts for packaging applications, and stopped using docker-in-docker in\ngitlab-runner.\n\n\nHere are a few of the main changes to my `.gitlab-ci.yml` file since my\nprevious post:\n\n\n* no docker-in-docker\n\n* using cache for packages instead of a prebuilt image with dependencies\n\n* splitting everything into multiple steps\n\n* autodeploy to staging environment using Helm, a package manager for\nKubernetes\n\n\n### Building Golang image\n\n\nSince Golang is very strict about the location of the project, we need to\nmake some adjustments to the CI job. This is done in the `before_script`\nblock. Simply create needed directories and link source code in there.\nAssuming that the official repository of the project is\n`gitlab.example.com/librerio/libr_files` it should look like this.\n\n\n```\n\nvariables:\n  APP_PATH: /go/src/gitlab.example.com/librerio/libr_files\n\nbefore_script:\n  - mkdir -p /go/src/gitlab.example.com/librerio/\n  - ln -s $PWD ${APP_PATH}\n  - mkdir -p ${APP_PATH}/vendor\n  - cd ${APP_PATH}\n```\n\n\nWith this in place, we can install dependencies and build our binaries. To\navoid the download of all packages on each build we need to configure\ncaching. Due to the strange caching rules of GitLab, we need to add vendor\ndirectory to both cache and artifacts. Cache will give us an ability to use\nit between build jobs and artifacts will allow us to use it inside the same\njob.\n\n\n```\n\n\ncache:\n  untracked: true\n  key: \"$CI_BUILD_REF_NAME\"\n  paths:\n    - vendor/\n\nsetup:\n  stage: setup\n  image: lwolf/golang-glide:0.12.3\n  script:\n    - glide install -v\n  artifacts:\n    paths:\n     - vendor/\n\n```\n\n\nBuild step didn’t change, it’s still about building the binary. I add binary\nto artifacts.\n\n\n```\n\nbuild:\n  stage: build\n  image: lwolf/golang-glide:0.12.3\n  script:\n    - cd ${APP_PATH}\n    - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o release/app -ldflags '-w -s'\n    - cd release\n  artifacts:\n    paths:\n     - release/\n```\n\n\n###  Test stage\n\n\nTo run golang tests with coverage reports I’m using the variation of [this\nshell\nscript](https://github.com/mlafeldt/chef-runner/blob/v0.7.0/script/coverage).\nIt runs all tests in project subdirectories and creates a [coverage\nreport](/blog/publish-code-coverage-report-with-gitlab-pages/). I changed it\na bit before putting into a gist. I exclude vendor directory from tests.\n\n\n* coverage regexp for gitlab-ci: `^total:\\s*\\(statements\\)\\s*(\\d+.\\d+\\%)`\n\n\n### Deploy stage\n\n\nI don’t use native GitLab’s integration with Kubernetes.\n\n\nFirst I thought about creating Kubernetes secrets and mounting it to the\ngitlab-runner pod. But it’s very complicated. You need to upgrade deployment\nevery time you want to add new Kubernetes cluster configurations. So I’m\nusing GitLab’s CI/CD variables with base64 encoded Kubernetes config. Each\nproject can have any number of configurations. The process is easy – create\nbase64 string from the configuration file and copy it to the clipboard.\nAfter this, put it into `kube_config` variable (name it whatever you like).\n\n\n`cat ~/.kube/config | base64 | pbcopy`\n\n\nIf you do not own a full GitLab installation, consider creating a Kubernetes\nuser with restricted permissions.\n\n\nThen on the deploy stage, we can decode this variable back into the file and\nuse it with kubectl.\n\n\n```\n\nvariables:\n  KUBECONFIG: /etc/deploy/config\n\ndeploy:\n  ...\n  before_script:\n    - mkdir -p /etc/deploy\n    - echo ${kube_config} | base64 -d > ${KUBECONFIG}\n    - kubectl config use-context homekube\n    - helm init --client-only\n    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/\n    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/\n    - helm repo update\n```\n\n\nDeploy stage also covers the case when you have several versions of the same\napplication.\n\n\nFor example, you have two versions of API: v1.0 and v1.1. All you need to do\nis set `appVersion` in Chart.yaml file. Build system will check API version\nand either deploy or upgrade needed release.\n\n\n```\n\n- export API_VERSION=\"$(grep \"appVersion\" Chart.yaml | cut -d\" \" -f2)\"\n\n- export RELEASE_NAME=\"libr-files-v${API_VERSION/./-}\"\n\n- export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)\n\n- if [ ${DEPLOYS}  -eq 0 ]; then helm install --name=${RELEASE_NAME} .\n--namespace=${STAGING_NAMESPACE}; else helm upgrade ${RELEASE_NAME} .\n--namespace=${STAGING_NAMESPACE}; fi\n\n```\n\n\n### tl;dr\n\n\n```\n\nHere is complete `.gitlab-ci.yaml` file for reference.\n\n\ncache:\n  untracked: true\n  key: \"$CI_BUILD_REF_NAME\"\n  paths:\n    - vendor/\n\nbefore_script:\n  - mkdir -p /go/src/gitlab.example.com/librerio/\n  - ln -s $PWD ${APP_PATH}\n  - mkdir -p ${APP_PATH}/vendor\n  - cd ${APP_PATH}\n\nstages:\n  - setup\n  - test\n  - build\n  - release\n  - deploy\n\nvariables:\n  CONTAINER_IMAGE: ${CI_REGISTRY}/${CI_PROJECT_PATH}:${CI_BUILD_REF_NAME}_${CI_BUILD_REF}\n  CONTAINER_IMAGE_LATEST: ${CI_REGISTRY}/${CI_PROJECT_PATH}:latest\n  DOCKER_DRIVER: overlay2\n\n  KUBECONFIG: /etc/deploy/config\n  STAGING_NAMESPACE: app-stage\n  PRODUCTION_NAMESPACE: app-prod\n\n  APP_PATH: /go/src/gitlab.example.com/librerio/libr_files\n  POSTGRES_USER: gorma\n  POSTGRES_DB: test-${CI_BUILD_REF}\n  POSTGRES_PASSWORD: gorma\n\nsetup:\n  stage: setup\n  image: lwolf/golang-glide:0.12.3\n  script:\n    - glide install -v\n  artifacts:\n    paths:\n     - vendor/\n\nbuild:\n  stage: build\n  image: lwolf/golang-glide:0.12.3\n  script:\n    - cd ${APP_PATH}\n    - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o release/app -ldflags '-w -s'\n    - cd release\n  artifacts:\n    paths:\n     - release/\n\nrelease:\n  stage: release\n  image: docker:latest\n  script:\n    - cd ${APP_PATH}/release\n    - docker login -u gitlab-ci-token -p ${CI_BUILD_TOKEN} ${CI_REGISTRY}\n    - docker build -t ${CONTAINER_IMAGE} .\n    - docker tag ${CONTAINER_IMAGE} ${CONTAINER_IMAGE_LATEST}\n    - docker push ${CONTAINER_IMAGE}\n    - docker push ${CONTAINER_IMAGE_LATEST}\n\ntest:\n  stage: test\n  image: lwolf/golang-glide:0.12.3\n  services:\n    - postgres:9.6\n  script:\n    - cd ${APP_PATH}\n    - curl -o coverage.sh https://gist.githubusercontent.com/lwolf/3764a3b6cd08387e80aa6ca3b9534b8a/raw\n    - sh coverage.sh\n\ndeploy_staging:\n  stage: deploy\n  image: lwolf/helm-kubectl-docker:v152_213\n  before_script:\n    - mkdir -p /etc/deploy\n    - echo ${kube_config} | base64 -d > ${KUBECONFIG}\n    - kubectl config use-context homekube\n    - helm init --client-only\n    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/\n    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/\n    - helm repo update\n  script:\n    - cd deploy/libr-files\n    - helm dep build\n    - export API_VERSION=\"$(grep \"appVersion\" Chart.yaml | cut -d\" \" -f2)\"\n    - export RELEASE_NAME=\"libr-files-v${API_VERSION/./-}\"\n    - export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)\n    - if [ ${DEPLOYS}  -eq 0 ]; then helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}; else helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}; fi\n  environment:\n    name: staging\n    url: https://librerio.example.com\n  only:\n  - master\n\n```\n\n\n_[How to create a CI/CD pipeline with Auto Deploy to Kubernetes using GitLab\nand\nHelm](http://blog.lwolf.org/post/how-to-create-ci-cd-pipeline-with-autodeploy-k8s-gitlab-helm/)\nwas originally published on Lwolfs Blog._\n\n\nPhoto by C Chapman on [Unsplash](https://unsplash.com/)\n",[787,788,9,2288],{"slug":2581,"featured":6,"template":700},"how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab","content:en-us:blog:how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab.yml","How To Create A Ci Cd Pipeline With Auto Deploy To Kubernetes Using Gitlab","en-us/blog/how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab.yml","en-us/blog/how-to-create-a-ci-cd-pipeline-with-auto-deploy-to-kubernetes-using-gitlab",{"_path":2587,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2588,"content":2594,"config":2602,"_id":2604,"_type":13,"title":2605,"_source":15,"_file":2606,"_stem":2607,"_extension":18},"/en-us/blog/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io",{"title":2589,"description":2590,"ogTitle":2589,"ogDescription":2590,"noIndex":6,"ogImage":2591,"ogUrl":2592,"ogSiteName":685,"ogType":686,"canonicalUrls":2592,"schema":2593},"Review Apps for Android with GitLab, fastlane & Appetize.io","See how GitLab and Appetize.io can bring Review Apps to your Android project","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664102/Blog/Hero%20Images/gitlab-values-cover.png","https://about.gitlab.com/blog/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to create Review Apps for Android with GitLab, fastlane, and Appetize.io\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Andrew Fontaine\"}],\n        \"datePublished\": \"2020-05-06\",\n      }",{"title":2595,"description":2590,"authors":2596,"heroImage":2591,"date":2598,"body":2599,"category":2600,"tags":2601},"How to create Review Apps for Android with GitLab, fastlane, and Appetize.io",[2597],"Andrew Fontaine","2020-05-06","{::options parse_block_html=\"true\" /}\n\n\n\n\nIn a [previous look at GitLab and _fastlane_], we discussed how _fastlane_\nnow\n\nautomatically publishes the Gitter Android app to the Google Play Store, but\nat\n\nGitLab, we live on [review apps], and review apps for Android applications\ndidn't\n\nreally exist... until [Appetize.io] came to our attention.\n\n\nJust a simple extension of our existing `.gitlab-ci.yml`, we can utilize\n\nAppetize.io to spin up review apps of our Android application.\n\n\nIf you'd rather just skip to the end, you can see\n\n[my MR to the Gitter Android project].\n\n\n## Setting up Fastlane\n\n\nFortunately for us, _fastlane_ has integrated support for Appetize.io, so\nall\n\nthat's needed to hit Appetize is the addition of a new `lane`:\n\n\n```diff\n\ndiff --git a/fastlane/Fastfile b/fastlane/Fastfile\n\nindex eb47819..f013a86 100644\n\n--- a/fastlane/Fastfile\n\n+++ b/fastlane/Fastfile\n\n@@ -32,6 +32,13 @@ platform :android do\n     gradle(task: \"test\")\n   end\n\n+  desc 'Pushes the app to Appetize and updates a review app'\n\n+  lane :review do\n\n+    appetize(api_token: ENV['APPETIZE_TOKEN'],\n\n+             path: 'app/build/outputs/apk/debug/app-debug.apk',\n\n+             platform: 'android')\n\n+  end\n\n+\n   desc \"Submit a new Internal Build to Play Store\"\n   lane :internal do\n     upload_to_play_store(track: 'internal', apk: 'app/build/outputs/apk/release/app-release.apk')\n```\n\n\n`APPETIZE_TOKEN` is an Appetize.io API token that can be generated on the\n\n[Appetize API docs] after signing up for an account. Once we add a new job\nand\n\nstage to our `.gitlab-ci.yml`, we will be able to deploy our APK to Appetize\nand\n\nrun them in the browser!\n\n\n```diff\n\ndiff --git a/.gitlab-ci.yml b/.gitlab-ci.yml\n\nindex d9863d7..e4d0ce3 100644\n\n--- a/.gitlab-ci.yml\n\n+++ b/.gitlab-ci.yml\n\n@@ -5,6 +5,7 @@ stages:\n   - environment\n   - build\n   - test\n+  - review\n   - internal\n   - alpha\n   - beta\n@@ -81,6 +82,16 @@ buildRelease:\n   environment:\n     name: production\n\n+deployReview:\n\n+  stage: review\n\n+  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG\n\n+  script:\n\n+    - bundle exec fastlane review\n\n+  only:\n\n+    - branches\n\n+  except:\n\n+    - master\n\n+\n testDebug:\n   image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG\n   stage: test\n```\n\n\nGreat! Review apps will be deployed when branches other than `master` build.\n\nUnfortunately, there is no `environment` block, so there's nothing linking\nthese\n\ndeployed review apps to GitLab. Let's fix that next.\n\n\n## Dynamic Environment URLs\n\n\nPreviously, GitLab only liked environment URLs that used pre-existing CI\n\nvariables (like `$CI_COMMT_REF_NAME`) in their definition. Since 12.9,\nhowever,\n\na [new way of defining environment urls with alternative variables exists].\n\n\nBy creating a `dotenv` file and submitting it as an `artifact` in our build,\nwe\n\ncan define custom variables to use in our environment's URL. As all\nAppetize.io\n\napp URLs take the pattern of `https://appetize.io.app/$PUBLIC_KEY`, where\n\n`$PUBLIC_KEY` is randomly generated when the app is created, we need to get\nthe\n\npublic key from the Appetize response in our `Fastfile`, and put it in a\n\n`dotenv` file.\n\n\n```diff\n\ndiff --git a/fastlane/Fastfile b/fastlane/Fastfile\n\nindex 7b5f9d1..ae3867c 100644\n\n--- a/fastlane/Fastfile\n\n+++ b/fastlane/Fastfile\n\n@@ -13,6 +13,13 @@\n # Uncomment the line if you want fastlane to automatically update itself\n # update_fastlane\n\n+\n\n+def update_deployment_url(pub_key)\n\n+  File.open('../deploy.env', 'w') do |f|\n\n+    f.write(\"APPETIZE_PUBLIC_KEY=#{pub_key}\")\n\n+  end\n\n+end\n\n+\n default_platform(:android)\n\n platform :android do\n@@ -37,6 +44,7 @@ platform :android do\n     appetize(api_token: ENV['APPETIZE_TOKEN'],\n              path: 'app/build/outputs/apk/debug/app-debug.apk',\n              platform: 'android')\n+    update_deployment_url(lane_context[SharedValues::APPETIZE_PUBLIC_KEY])\n   end\n\n   desc \"Submit a new Internal Build to Play Store\"\n```\n\n\nWe also need to add an `environment` block to our `.gitlab-ci.yml` to\ncapture an\n\nenvironment name and URL.\n\n\n```diff\n\ndiff --git a/.gitlab-ci.yml b/.gitlab-ci.yml\n\nindex f5a8648..c834077 100644\n\n--- a/.gitlab-ci.yml\n\n+++ b/.gitlab-ci.yml\n\n@@ -85,12 +85,18 @@ buildCreateReleaseNotes:\n deployReview:\n   stage: review\n   image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG\n+  environment:\n\n+    name: review/$CI_COMMIT_REF_NAME\n\n+    url: https://appetize.io/app/$APPETIZE_PUBLIC_KEY\n   script:\n     - bundle exec fastlane review\n   only:\n     - branches\n   except:\n     - master\n+  artifacts:\n\n+    reports:\n\n+      dotenv: deploy.env\n\n testDebug:\n   image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG\n```\n\n\nOnce committed, pushed, and a pipeline runs, we should see our environment\n\ndeployed!\n\n\n![Our first review environment][first-review-app]\n\n\n## Optimizing Updates\n\n\nAfter running with this for a bit, we realized that we were accidentally\n\ncreating a new app on Appetize.io with every new build! Their docs\n\n[specify how to update existing apps], so we went about seeing if we could\n\nsmartly update existing environments.\n\n\nSpoiler alert: We could.\n\n\nFirst, we need to save the public key granted to us by Appetize.io\nsomewhere. We\n\ndecided to put it in a JSON file and save that as an artifact of the build.\n\nFortunately, the `Fastfile` is just ruby, which allows us to quickly write\nit\n\nout to a file with a few lines of code, as well as attempt to fetch the\nartifact\n\nfor the last build of the current branch.\n\n\n```diff\n\ndiff --git a/fastlane/Fastfile b/fastlane/Fastfile\n\nindex ae3867c..61e9226 100644\n\n--- a/fastlane/Fastfile\n\n+++ b/fastlane/Fastfile\n\n@@ -13,8 +13,32 @@\n # Uncomment the line if you want fastlane to automatically update itself\n # update_fastlane\n\n+require 'net/http'\n\n+require 'json'\n\n+\n\n+GITLAB_TOKEN = ENV['PRIVATE_TOKEN']\n\n+PROJECT_ID = ENV['CI_PROJECT_ID']\n\n+REF = ENV['CI_COMMIT_REF_NAME']\n\n+JOB = ENV['CI_JOB_NAME']\n\n+API_ROOT = ENV['CI_API_V4_URL']\n\n+\n\n+def public_key\n\n+  uri =\nURI(\"#{API_ROOT}/projects/#{PROJECT_ID}/jobs/artifacts/#{REF}/raw/appetize-information.json?job=#{JOB}\")\n\n+  http = Net::HTTP.new(uri.host, uri.port)\n\n+  http.use_ssl = true\n\n+  req = Net::HTTP::Get.new(uri)\n\n+  req['PRIVATE-TOKEN'] = GITLAB_TOKEN\n\n+  response = http.request(req)\n\n+  return '' if response.code.equal?('404')\n\n+\n\n+  appetize_info = JSON.parse(response.body)\n\n+  appetize_info['publicKey']\n\n+end\n\n def update_deployment_url(pub_key)\n+  File.open('../appetize-information.json', 'w') do |f|\n\n+    f.write(JSON.generate(publicKey: pub_key))\n\n+  end\n   File.open('../deploy.env', 'w') do |f|\n     f.write(\"APPETIZE_PUBLIC_KEY=#{pub_key}\")\n   end\n@@ -42,6 +66,7 @@ platform :android do\n   desc 'Pushes the app to Appetize and updates a review app'\n   lane :review do\n     appetize(api_token: ENV['APPETIZE_TOKEN'],\n+             public_key: public_key,\n              path: 'app/build/outputs/apk/debug/app-debug.apk',\n              platform: 'android')\n     update_deployment_url(lane_context[SharedValues::APPETIZE_PUBLIC_KEY])\n```\n\n\nWhen we go to deploy our app to Appetize, we hit the [Jobs API] to see if we\n\nhave a public key for this branch. If the API returns a `404`, we know we\nare\n\nbuilding a fresh branch and return an empty string, else we parse the JSON\nand\n\nreturn our public key. The [Fastlane docs] state the `appetize` action can\ntake\n\na `public_key` to update an existing app. Here, `''` is considered the same\nas\n\n_not_ providing a public key, so a new application is still deployed as we\nexpect.\n\n\n**NOTE:** If you've read the `diff` closely, you'll notice the usage of an\n\nenvironment variable called `PRIVATE_TOKEN`. This is a GitLab private token\n\ncreated with the `read_api` scope and injected into our build as an\nenvironment\n\nvariable. This is required to authenticate with the GitLab API and fetch\n\nartifacts.\n\n\nOnce we update `.gitlab-ci.yml` to save the new `appetize-information.json`\nfile\n\nas an artifact, later builds on the same branch will be smart and update the\n\nexisting Appetize app!\n\n\n```diff\n\ndiff --git a/.gitlab-ci.yml b/.gitlab-ci.yml\n\nindex c834077..54cf3f6 100644\n\n--- a/.gitlab-ci.yml\n\n+++ b/.gitlab-ci.yml\n\n@@ -95,6 +95,8 @@ deployReview:\n   except:\n     - master\n   artifacts:\n+    paths:\n\n+      - appetize-information.json\n     reports:\n       dotenv: deploy.env\n```\n\n\n## Cleaning up\n\n\nAll that's left is to delete old apps from Appetize once we don't need them\n\nanymore. We can do that by leveraging `on_stop` and creating a `stop` job\nthat\n\nwill delete our app from Appetize.io\n\n\n```diff\n\ndiff --git a/.gitlab-ci.yml b/.gitlab-ci.yml\n\nindex 54cf3f6..f6ecf7e 100644\n\n--- a/.gitlab-ci.yml\n\n+++ b/.gitlab-ci.yml\n\n@@ -10,6 +10,7 @@ stages:\n   - alpha\n   - beta\n   - production\n+  - stop\n\n\n .updateContainerJob:\n@@ -88,6 +89,7 @@ deployReview:\n   environment:\n     name: review/$CI_COMMIT_REF_NAME\n     url: https://appetize.io/app/$APPETIZE_PUBLIC_KEY\n+    on_stop: stopReview\n   script:\n     - bundle exec fastlane review\n   only:\n@@ -100,6 +102,22 @@ deployReview:\n     reports:\n       dotenv: deploy.env\n\n+stopReview:\n\n+  stage: stop\n\n+  environment:\n\n+    name: review/$CI_COMMIT_REF_NAME\n\n+    action: stop\n\n+  variables:\n\n+    GIT_STRATEGY: none\n\n+  when: manual\n\n+  only:\n\n+    - branches\n\n+  except:\n\n+    - master\n\n+  script:\n\n+    - apt-get -y update && apt-get -y upgrade && apt-get -y install jq curl\n\n+    - curl --request DELETE\nhttps://$APPETIZE_TOKEN@api.appetize.io/v1/apps/`jq -r '.publicKey' \u003C\nappetize-information.json`\n\n+\n testDebug:\n   image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG\n   stage: test\n```\n\n\nOnce your MR is merged and your branch is deleted, the `stopReview` job\nruns,\n\ncalling the [`DELETE` endpoint of the Appetize.io API] with the public key\nthat\n\nis contained in `appetize-information.json`. We don't need to fetch\n\n`appetize-information.json` because the artifact is already present in our\nbuild\n\ncontext. This is because the `stop` stage happens _after_ the `review`\nstage.\n\n\n![A merge request with a deployed review app][merge-request-with-review-app]\n\n\n## Conclusion\n\n\nThanks to some integration with _fastlane_ and the addition of a couple\n\nenvironment variables, having the ability to create review apps for an\nAndroid\n\nproject was surpsingly simple. GitLab's review apps are not _just_ for\nweb-based\n\nprojects, even though it may take a little tinkering to get working.\nAppetize.io\n\nalso supports iOS applications, so all mobile native applications can be\nturned\n\ninto review apps. I would love to see this strategy be applied to a React\nNative\n\nproject as well!\n\n\n[previous look at gitlab and _fastlane_]:\n/blog/android-publishing-with-gitlab-and-fastlane/\n\n[my mr to the gitter android project]:\nhttps://gitlab.com/gitlab-org/gitter/gitter-android-app/-/merge_requests/167\n\n[review apps]: https://docs.gitlab.com/ee/ci/review_apps/#review-apps\n\n[appetize.io]: https://appetize.io\n\n[appetize api docs]: https://appetize.io/docs#request-api-token\n\n[new way of defining environment urls with alternative variables exists]:\nhttps://docs.gitlab.com/ee/ci/environments/index.html#set-dynamic-environment-urls-after-a-job-finishes\n\n[first-review-app]:\n/images/blogimages/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io/first-review-app.png\n\n[specify how to update existing apps]:\nhttps://appetize.io/docs#updating-apps\n\n[jobs api]:\nhttps://docs.gitlab.com/ee/api/jobs.html#download-a-single-artifact-file-from-specific-tag-or-branch\n\n[fastlane docs]: https://docs.fastlane.tools/actions/appetize/\n\n[`delete` endpoint of the appetize.io api]:\nhttps://appetize.io/docs#deleting-apps\n\n[merge-request-with-review-app]:\n/images/blogimages/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io/merge-request-with-review-app.png\n","unfiltered",[108,232,696,9],{"slug":2603,"featured":6,"template":700},"how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io","content:en-us:blog:how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io.yml","How To Create Review Apps For Android With Gitlab Fastlane And Appetize Dot Io","en-us/blog/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io.yml","en-us/blog/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io",{"_path":2609,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2610,"content":2616,"config":2621,"_id":2623,"_type":13,"title":2624,"_source":15,"_file":2625,"_stem":2626,"_extension":18},"/en-us/blog/how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration",{"title":2611,"description":2612,"ogTitle":2611,"ogDescription":2612,"noIndex":6,"ogImage":2613,"ogUrl":2614,"ogSiteName":685,"ogType":686,"canonicalUrls":2614,"schema":2615},"How to deploy a PHP app using GitLab's Cloud Run integration","Are you using PHP and want an easy way to deploy your application to Google Cloud? Follow this guide to deploy your app with Google Cloud Run in under 10 minutes.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098264/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_519147119_2RafH61mqosMZv8HGAlsUj_1750098264407.jpg","https://about.gitlab.com/blog/how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to deploy a PHP app using GitLab's Cloud Run integration\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Christian Nnachi\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2024-12-10\",\n      }",{"title":2611,"description":2612,"authors":2617,"heroImage":2613,"date":2618,"body":2619,"category":785,"tags":2620},[1363,847],"2024-12-10","Writing PHP application code and ensuring the application is running\nsmoothly in production are often two different skills sets owned by two\ndifferent engineers. GitLab aims to bridge the gap by enabling the engineer\nwho has written the PHP application code to also deploy it into Google Cloud\nPlatform with little effort. \n\n\nWhether you own event-driven, long-running services or deploy containerized\njobs to process data, Google Cloud Run automatically scales your containers\nup and down from zero — this means you only pay when your code is running.\n\n\nIf you are a PHP developer who would like to deploy your application with\nminimal effort to Google Cloud Platform, this guide will show you how using\nthe GitLab Google Cloud Run integration. \n\n\n# Overview\n\n\n- Create a new project in GitLab\n\n- Set up your PHP application\n\n- Utilizing the Google Cloud integration, create a Service account\n\n- Utilizing the Google Cloud integration, configure Cloud Run via merge\nrequest\n\n- Try adding another endpoint\n\n- Clean up\n\n\n## Prerequisites\n\n- Owner access on a Google Cloud Platform project\n\n- Working knowledge of\n[PHP](https://www.php.net/manual/en/introduction.php), an open-source,\ngeneral-purpose scripting language\n\n- Working knowledge of [GitLab\nCI](https://about.gitlab.com/topics/ci-cd/#what-is-continuous-integration-ci)\n\n- 10 minutes\n\n\n## 1. Create a new project in GitLab.\n\n\nWe decided to call our project `PHP cloud-run` for simplicity.\n\n\n![PHP cloud- run\nproject](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098287/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098287615.png)\n\n\nThen, create an index.php\napp[https://gitlab.com/demos/templates/php-cloud-run/-/blob/main/index.php](https://gitlab.com/demos/templates/php-cloud-run/-/blob/main/index.php).\n\n\n```php\n\n\u003C?php\n\n\n$name = getenv('NAME', true) ?: 'World';\n\necho sprintf('Hello %s!', $name);\n\n```\n\n\n## 2. Utilizing the Google Cloud integration, create a Service account.\n\n\nNavigate to **Operate > Google Cloud > Create Service account**. \n\n\n![Create Service account\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098287616.png)\n\n\nThen configure the region you would like the Cloud Run instance deployed to.\n\n\n![Configure region\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098287618.png)\n\n\n## 3. Utilizing the Google Cloud integration, configure **Cloud Run via\nmerge request**.\n\n\n![Deployment configuration\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098287620.png)\n\n\nThis will open a merge request. Immediately merge this merge request.\n\n\n![Enable Deployments to Cloud run\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098287622.png)\n\n\n**Note:** `GCP_PROJECT_ID`, `GCP_REGION`,  `GCP_SERVICE_ACCOUNT`, and\n`GCP_SERVICE_ACCOUNT_KEY` will all be automatically populated from the\nprevious steps.\n\n\n![Variables\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098287624.png)\n\n\nCheck your pipeline and you will see you have successfully deployed to\nGoogle Cloud Run utilizing GitLab CI.\n\n\n![merge branch\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098287625.png)\n\n\n\u003Cbr>\u003C/br>\n\n\n![Google Cloud Run deployed with GitLab\nCI](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098287627.png)\n\n\n## 4. Click the **Service URL** to view your newly deployed Flask server.\n\n\nIn addition, you can navigate to **Operate > Environments** to see a list of\ndeployments for your environments.\n\n\n![Environments\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098287628.png)\n\n\nBy clicking on the environment called **main**, you’ll be able to view a\ncomplete list of deployments specific to that environment.\n\n\n![Main\nenvironment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098288/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098287631.png)\n\n\n## 5. Add another endpoint\n\n\nTo get started with developing your PHP application, try adding another\nendpoint. For example, in your main file, you can add a `/bye` endpoint like\nthis:\n\n\n```\n\n\n\u003C?php\n\n\n$name = getenv('NAME', true) ?: 'World';\n\n\nif ($_SERVER['REQUEST_URI'] == '/bye') {\n    echo sprintf('Goodbye %s!', $name);\n} else {\n    echo sprintf('Hello %s!', $name);\n}\n\n\n```\n\n\nPush the changes to the repo, and watch the `deploy-to-cloud-run` job deploy\nthe updates. Once the job is complete, go back to the Service URL and\nnavigate to the `/bye` endpoint to see the new functionality in action.\n\n\n### Clean up\n\n\nTo prevent incurring charges on your Google Cloud account for the resources\nused in this tutorial, you can either delete the specific resources or\ndelete the entire Google Cloud project. For detailed instructions, refer to\nthe [cleanup guide\nhere](https://docs.gitlab.com/ee/tutorials/create_and_deploy_web_service_with_google_cloud_run_component/#clean-up).\n\n\n> Check out more [easy-to-follow tutorials from our Solutions Architecture\nteam](https://about.gitlab.com/blog/tags/solutions-architecture/).\n",[1016,9,1264,232],{"slug":2622,"featured":6,"template":700},"how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration","content:en-us:blog:how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration.yml","How To Deploy A Php App Using Gitlabs Cloud Run Integration","en-us/blog/how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration.yml","en-us/blog/how-to-deploy-a-php-app-using-gitlabs-cloud-run-integration",{"_path":2628,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2629,"content":2635,"config":2641,"_id":2643,"_type":13,"title":2644,"_source":15,"_file":2645,"_stem":2646,"_extension":18},"/en-us/blog/how-to-detecting-secrets-in-video",{"title":2630,"description":2631,"ogTitle":2630,"ogDescription":2631,"noIndex":6,"ogImage":2632,"ogUrl":2633,"ogSiteName":685,"ogType":686,"canonicalUrls":2633,"schema":2634},"How-to: Detecting secrets in video content ","GitLab’s Security team identifies and mitigates security risks in video content by searching for API keys or other sensitive tokens. Here's how we do it (with an assist from AI) and how you can, too.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099421/Blog/Hero%20Images/Blog/Hero%20Images/security-checklist_security-checklist.png_1750099421443.png","https://about.gitlab.com/blog/how-to-detecting-secrets-in-video-content","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How-to: Detecting secrets in video content \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Dennis Appelt\"}],\n        \"datePublished\": \"2024-02-29\",\n      }",{"title":2630,"description":2631,"authors":2636,"heroImage":2632,"date":2638,"body":2639,"category":830,"tags":2640},[2637],"Dennis Appelt","2024-02-29","Today, we are open-sourcing our [solution for detecting secrets in video content]( https://gitlab.com/gitlab-com/gl-security/security-research/video-scanner/youtube-video-scanner). We use it internally to search videos published on our [GitLab Unfiltered YouTube channel](https://www.youtube.com/@GitLabUnfiltered) for [secrets](https://docs.gitlab.com/ee/security/token_overview.html) such as API keys and other sensitive tokens.\n\nWhile there are existing tools for secret detection, we did not find a tool that quite fit the bill for our use case, so we decided to implement a custom scanner. In this blog post, we'll walk through our general approach, some of the challenges we encountered, and our solution. We'll also discuss how GitLab’s new AI assistant, [GitLab Duo Chat](https://about.gitlab.com/gitlab-duo/), helped with the implementation of the scanner.\n\n## Scanning videos, one frame at a time\n\nOur general approach to secret detection in videos is quite simple: Split the video into frames, run optical character recognition (OCR) over each frame, and match the resulting text against known [secret patterns](https://docs.gitlab.com/ee/security/token_overview.html). If a secret is found, a [security incident](https://handbook.gitlab.com/handbook/security/security-operations/sirt/engaging-security-on-call/#engage-the-security-engineer-on-call) is kicked off to investigate the leak and revoke exposed secrets.\n\nTo implement this approach, we first experimented using [FFmpeg](https://ffmpeg.org//) for splitting the video into frames and feeding the frames to [Tesseract](https://github.com/tesseract-ocr/tesseract), an open-source engine for OCR. This worked quite well and gave us confidence that the general approach was feasible. However, we decided to switch to [Google Cloud Platform's Video Intelligence API](https://cloud.google.com/video-intelligence/docs/) for the frame splitting and OCR for the simple reason of not having to scale and maintain our own implementation.\n\nFFmpeg and Tesseract are good options if third-party APIs cannot be used or if more control over the process is required. For example, if the secrets are only exposed for a brief moment in the video, using FFmpeg allows you to increase the frame sampling rate to analyze more frames per second and increases the chances of catching the frame that exposes the secret. The Video Intelligence API does not provide a comparable level of control. \n\nThe choice between the Video Intelligence API and FFmpeg + Tesseract also depends on the data set that has to be analyzed. The Video Intelligence API works well on our data set, which makes the additional complexity of a custom implementation based on FFmpeg + Tesseract hard to justify. After settling for the Video Intelligence API, it was a natural choice to host the rest of the scanner on GCP as well. The below diagram gives an overview of the design:\n\n![video content scanners - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099431138.png)\n\nThe scanner is implemented as a collection of cloud functions running on GCP. The cloud function `WebSub API` implements the WebSub [spec](https://www.w3.org/TR/websub/), which is used by YouTube to deliver notifications. Notifications of new videos are published to a PubSub topic, which the cloud function `Video Fetcher` is subscribed to. If a message is received, the video is downloaded and submitted for OCR to the Video Intelligence API. The resulting text extract is checked for secrets by the `Secret Matcher` and alerts are created in case a secret is found.\n\n## Accounting for inaccuracies in OCR\n\nThe described approach sounds simple enough, but as with most things, the devil is in the details. When comparing the video scanner to other secret scanning methods, a notable difference is how the video scanner determines if a given string literal is a secret. Secret detection tools usually determine if the given text contains a secret by matching the text against a list of regular expressions, each defining the format of a secret. If there is a match, a secret is detected.\n\n![video content scanners - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099431139.png)\n\n\u003Ccenter>\u003Ci>A video frame showing a GitLab access token\u003C/i>\u003C/center>\n\n\u003Cp>\u003C/p>\n\nWhen it comes to video scanning, this approach has limited effectiveness due to the OCR step. In some instances, the recognized text does not quite match the text displayed in the video. For example, the above video frame shows the access token `glpat-HseyLLLE92Ubso2vyVeD` and OCR extracted the text `glpat-HseyLLLE92Ubso2vyVe\\`. The last character of the secret is `D`, but OCR extracted a backslash ( `\\`). This error causes the extracted text to no longer match the format of GitLab personal access tokens; therefore, simply matching the text against a regular expression conforming to the token format would have not detected the leaked access token.\n\nTo account for the inaccuracies that are introduced by the OCR step, the video scanner uses approximate regular expression matching where a string is not required to match a regular expression exactly, but small deviations in the strings are allowed. These deviations are expressed as string edit distance and define how many characters in the string need to be inserted, deleted, or substituted to make the string match a given regular expression. For example, the string edit distance for the previous example is 1 because the erroneously detected backslash has to be substituted with an alphanumeric character or a minus sign to make the string match the GitLab personal access token format.\n\nFinding the right value for the maximum edit distance for a string to still be considered to match a regular expression depends on the data set and requires some experimentation. If the value is too low, secrets might be missed and if the value is too high, strings that are not secrets will be matched. To find the right balance for our use case, we collected a data set of videos leaking secrets and ran them through our scanner pipeline. We measured precision (how many matches are not secrets) and recall (how many matches are relevant/true secrets) for different maximum edit distances. We collected this data for the two most relevant types of secrets: GitLab API access tokens and runner registration tokens. The below diagrams show the results.\n\nFor both types of secrets, the optimal maximum cost is 2, at which point a high precision and recall are achieved (97% precision and 91% recall for access tokens; 100% precision and 86% recall for runner registration tokens). Decreasing the maximum distance results in worse recall, while increasing the maximum cost leads to either no improvement (runner registration token) or adversely affects precision (access tokens).\n\n![video content scanner - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099431141.png)\n\n\u003Cp>\u003C/p>\n\n![video content scanner - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099431143.png)\n\n## Implementation made easy using GitLab Duo Chat\n\nAfter some initial experimentation, it was clear to us that approximate regular expression matching can increase the detection accuracy of our video scanner. However, the implementation turned out not to be trivial. While the [algorithms for approximate regular expression matching](https://www.tgries.de/agrep/) have existed since the '90s, there is no widespread library support. [TRE](https://laurikari.net/tre/) is a solid choice but is written in C and thus requires bindings to be called from the scanner’s runtime environment (Node.js).\n\nTo reduce the development effort, we turned to [GitLab Duo Chat](https://about.gitlab.com/gitlab-duo/). After discussing our options with Duo Chat for calling C from Node.js, we settled on [implementing a C++ add-on](https://nodejs.org/docs/latest-v20.x/api/addons.html). We quickly iterated towards a working implementation, which would have undoubtedly taken considerably more time without the help of GitLab Duo Chat. Below is an excerpt from the chat session.\n\n> Me: Are you familiar with the C library TRE? Can you help me write a Node.js addon that is a wrapper for TRE?\n\n> Duo: Here is the template for wrapping the TRE library in a Node.js addon using N-API:\n>\n>![video content scanner - image 5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-02-28_at_9.51.14_AM_aHR0cHM6_1750099431144.png)\n>\n> The key steps are:\n> - Include TRE header\n> - Get tre regex_t from parameters\n> - Use tre_regcomp() to compile pattern\n> - Execute regexec()\n> - Handle TRE match results\n> - Return results back to JavaScript\n>\n> Let me know if this helps explain how to integrate TRE regex usage in a Node.js add-on!\n\n> Me: Ok, now let's add code for the tasks that you outlined in the template. I want to call this C++ from JavaScript like this:\n>\n>![video content scanners - image 6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-02-28_at_9.53.05_AM_aHR0cHM6_1750099431145.png)\n>\n> In this example, the first parameter is the regular expression that should be used to create the parameter regex_t *preg, the second parameter is the string that should be matched against the regular expression, and the third parameter is an integer that represents TRE’s max_cost.\n\n>Duo: Here is example C++ code implementing TRE regex usage to match the provided JavaScript code snippet.\n>\n>![video content scanners - image 7](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-02-28_at_9.54.08_AM_aHR0cHM6_1750099431146.png)\n>\n> Let me know if you need any clarification on this implementation!\n\n> Me: Thank you for the code. Can you change the call tre_regexec to tre_regaexec?\n\n> Duo: Here is the code updated to use tre_regaexec instead of tre_regexec:\n>\n>![code updated to use tre_regaexec instead of tre_regexec](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099431/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-02-28_at_9.56.29_AM_aHR0cHM6_1750099431146.png)\n>\n> The key changes are:\n> - Using `tre_regaexec` instead of `tre_regexec`\n> - Initializing a `regamatch_t` instead of `regmatch_t`\n> - Passing a regaparams_t struct to configure match parameters\n> \n> Let me know if you need any clarification or have additional questions!\n\nWhile the code generation capabilities are very impressive and save a lot of time, the generated code needs review and sometimes correction. For example, in our session with Duo Chat it is prompted to call [tre_regaexec](https://laurikari.net/tre/documentation/regaexec/), but the generated code is calling the similarly named, but distinct function [tre_regexec](https://laurikari.net/tre/documentation/regexec/). Typically it is enough to make Duo Chat aware of its mistake and it will correct the code in question. In this respect, working with Duo Chat feels like pair programming rather than fully outsourcing a programming task.\n\nThe full add-on code is available [here](https://gitlab.com/gitlab-com/gl-security/security-research/video-scanner/tre-node-bindings/).\n\n## Try our open-source implementation\n\nWe are [making the implementation of the scanner open source](https://gitlab.com/gitlab-com/gl-security/security-research/video-scanner/youtube-video-scanner) under the MIT license. We hope this solution can help you with detecting secrets in your own video content. Please [share your feedback in an issue](https://gitlab.com/gitlab-com/gl-security/security-research/video-scanner/youtube-video-scanner/-/issues/new).",[9,743,830],{"slug":2642,"featured":90,"template":700},"how-to-detecting-secrets-in-video-content","content:en-us:blog:how-to-detecting-secrets-in-video.yml","How To Detecting Secrets In Video","en-us/blog/how-to-detecting-secrets-in-video.yml","en-us/blog/how-to-detecting-secrets-in-video",{"_path":2648,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2649,"content":2655,"config":2661,"_id":2663,"_type":13,"title":2664,"_source":15,"_file":2665,"_stem":2666,"_extension":18},"/en-us/blog/how-to-fuzz-rust-code",{"title":2650,"description":2651,"ogTitle":2650,"ogDescription":2651,"noIndex":6,"ogImage":2652,"ogUrl":2653,"ogSiteName":685,"ogType":686,"canonicalUrls":2653,"schema":2654},"How to fuzz Rust code continuously","Learn why you should always fuzz test your Rust code, and the code you'll need to do it.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749681441/Blog/Hero%20Images/rust.jpg","https://about.gitlab.com/blog/how-to-fuzz-rust-code","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to fuzz Rust code continuously\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Yevgeny Pats\"}],\n        \"datePublished\": \"2020-12-03\",\n      }",{"title":2650,"description":2651,"authors":2656,"heroImage":2652,"date":2658,"body":2659,"category":785,"tags":2660},[2657],"Yevgeny Pats","2020-12-03","\n\nThis blog post was originally published on the GitLab Unfiltered blog. It was reviewed and republished on 2020-12-17.\n{: .alert .alert-info .note}\n\n## What is fuzzing?\n\nFuzzing, also called [fuzz testing](/topics/devsecops/what-is-fuzz-testing/), is an automated software technique that involves providing semi-random data as an input to the test program in order to uncover bugs and crashes.\n\nIn this short tutorial we will discuss using `cargo-fuzz` for fuzzing Rust code.\n\n## Why fuzz Rust code?\n[Rust](https://www.rust-lang.org/) is a safe language (mostly) and memory corruption issues are a thing of the past so we don’t need to fuzz our code, right? Wrong!\nAny code, and especially where stability, quality, and coverage are important, is worth fuzzing.\nFuzzing can uncover logical bugs and denial-of-service issues in critical components that can lead to security issues as well.\n\nAs a reference to almost infinite amount of bugs found with cargo-fuzz (only the documented one) you can look at [the list of bugs found by fuzz-testing Rust codebases](https://github.com/rust-fuzz/trophy-case).\n\n## Cargo-fuzz\n\ncargo-fuzz is the current de-facto standard fuzzer for Rust and essentially it is a proxy layer to the well-tested [libFuzzer](https://llvm.org/docs/LibFuzzer.html) engine.\nThis means the algorithm and the interface is all based on libFuzzer, which is a widely-used, coverage-guided fuzzer for C/C++ and some other languages that implemented a proxy layer – just like cargo-fuzz.\n\nlibFuzzer (cargo-fuzz) and coverage-guided fuzzers in general have the following algorithm:\n\n```\n// pseudo code\nInstrument program for code coverage\nfor {\n  Choose random input from corpus\n  Mutate input\n  Execute input and collect coverage\n  If new coverage/paths are hit add it to corpus (corpus - directory with test-cases)\n}\n```\n\n## Building and running the fuzzer\n\nIf you are already familiar with this part you can skip to Continuous Fuzzing section.\n\nWe will start with [rust-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example).\n\nFor the sake of the example, we have a simple [function](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example/-/blob/master/src/lib.rs) with an off-by-one bug:\n\n```\npub fn parse_complex(data: &[u8]) -> bool{\n\tif data.len() == 5 {\n\t\tif data[0] == b'F' && data[1] == b'U' && data[2] == b'Z' && data[3] == b'Z' && data[4] == b'I' && data[5] == b'T' {\n\t\t\treturn true\n\t\t}\n\t}\n    return true;\n}\n```\n\nOur fuzz [function](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example/-/blob/master/fuzz/fuzz_targets/fuzz_parse_complex.rs) will look like this and will be called by libFuzzer in an infinite loop with the generated data, according to the coverage-guided algorithm.\n\n```\n#![no_main]\n#[macro_use] extern crate libfuzzer_sys;\nextern crate example_rust;\n\nfuzz_target!(|data: &[u8]| {\n    let _ = example_rust::parse_complex(&data);\n});\n```\n\nTo run the fuzzer we need to build an instrumented version of the code together with the fuzz function.\ncargo-fuzz is doing for us the heavy lifting so it can be done using the following simple steps:\n\n```\n# cargo-fuzz is available in rust nightly\ndocker run -it rustlang/rust:nightly-stretch /bin/bash\ncargo install cargo-fuzz\n\n# Download the example repo, build, and run the fuzzer\ngit clone https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example/-/blob/master/fuzz/fuzz_targets/fuzz_parse_complex.rs\ncd example-rust\ncargo fuzz run fuzz_parse_complex\n\n## The output should look like this:\n#524288 pulse  cov: 105 ft: 99 corp: 6/26b lim: 517 exec/s: 131072 rss: 93Mb\n#1048576        pulse  cov: 105 ft: 99 corp: 6/26b lim: 1040 exec/s: 116508 rss: 229Mb\n==2208== ERROR: libFuzzer: deadly signal\n    #0 0x5588b8234961  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x83961)\n    #1 0x5588b8262dc5  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xb1dc5)\n    #2 0x5588b8284734  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xd3734)\n    #3 0x5588b82845e9  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xd35e9)\n    #4 0x5588b826493a  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xb393a)\n    #5 0x7f93737e70df  (/lib/x86_64-linux-gnu/libpthread.so.0+0x110df)\n    #6 0x7f9373252ffe  (/lib/x86_64-linux-gnu/libc.so.6+0x32ffe)\n    #7 0x7f9373254429  (/lib/x86_64-linux-gnu/libc.so.6+0x34429)\n    #8 0x5588b82a4a06  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf3a06)\n    #9 0x5588b82a1b75  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf0b75)\n    #10 0x5588b824fa1b  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x9ea1b)\n    #11 0x5588b82a442b  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf342b)\n    #12 0x5588b82a3ee1  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf2ee1)\n    #13 0x5588b82a3dd5  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf2dd5)\n    #14 0x5588b82b6cd9  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x105cd9)\n    #15 0x5588b82b6c94  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x105c94)\n    #16 0x5588b824edda  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x9ddda)\n    #17 0x5588b81c45b7  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x135b7)\n    #18 0x5588b824f7e4  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0x9e7e4)\n    #19 0x5588b827da53  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xcca53)\n    #20 0x5588b82a4a18  (/example-rust/fuzz/target/x86_64-unknown-linux-gnu/debug/fuzz_parse_complex+0xf3a18)\n\nNOTE: libFuzzer has rudimentary signal handlers.\n      Combine libFuzzer with AddressSanitizer or similar for better crash reports.\nSUMMARY: libFuzzer: deadly signal\nMS: 2 ShuffleBytes-ChangeByte-; base unit: 89b92cdd9bcb9b861c47c0179eff7b3a9baafcde\n0x46,0x55,0x5a,0x5a,0x49,\nFUZZI\nartifact_prefix='/example-rust/fuzz/artifacts/fuzz_parse_complex/'; Test unit written to /example-rust/fuzz/artifacts/fuzz_parse_complex/crash-df779ced6b712c5fca247e465de2de474d1d23b9\nBase64: RlVaWkk=\n```\n\nThis find the bug in a few seconds, prints the “FUZZI” string that triggers the vulnerability and saves it to a file.\n\n## Running cargo-fuzz from CI\n\nThe best way to integrate go-fuzz fuzzing with Gitlab CI/CD is by adding additional stage and step to your `.gitlab-ci.yml`. It is straightforward and [fully documented](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#configuration).\n\n```\ninclude:\n  - template: Coverage-Fuzzing.gitlab-ci.yml\n\nmy_fuzz_target:\n  extends: .fuzz_base\n  script:\n    - apt-get update -qq && apt-get install -y -qq git make clang cmake\n    - export CC=`which clang`\n    - export CXX=`which clang++`\n    - cargo install cargo-fuzz\n    - cargo fuzz run fuzz_parse_complex -- -runs=0\n    - ./gitlab-cov-fuzz run --regression=$REGRESSION -- ./fuzz/target/x86_64-unknown-linux-gnu/release/fuzz_parse_complex\n```\n\nFor each fuzz target you will have to create a step which extends `.fuzz_base` that runs the following:\n\n- Builds the fuzz target.\n- Runs the fuzz target via gitlab-cov-fuzz CLI.\n- For `$CI_DEFAULT_BRANCH` (can be override by `$COV_FUZZING_BRANCH`) will run fully fledged fuzzing sessions. For everything else including MRs will run fuzzing regression with the accumulated corpus and fixed crashes.\n\nThis will run your fuzz tests in a blocking manner inside your pipeline. There is also a possibility to run longer fuzz sessions asynchronously, as described in the [docs](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#continuous-fuzzing-long-running-async-fuzzing-jobs).\n\nCheck out our [full documentation](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/) and the [example repo](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example) and try adding fuzz testing to your own repos!\n\nCover image by [Zsolt Palatinus](https://unsplash.com/@sunitalap) on [Unsplash](https://unsplash.com/)\n",[743,9],{"slug":2662,"featured":6,"template":700},"how-to-fuzz-rust-code","content:en-us:blog:how-to-fuzz-rust-code.yml","How To Fuzz Rust Code","en-us/blog/how-to-fuzz-rust-code.yml","en-us/blog/how-to-fuzz-rust-code",{"_path":2668,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2669,"content":2675,"config":2681,"_id":2683,"_type":13,"title":2684,"_source":15,"_file":2685,"_stem":2686,"_extension":18},"/en-us/blog/how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id",{"title":2670,"description":2671,"ogTitle":2670,"ogDescription":2671,"noIndex":6,"ogImage":2672,"ogUrl":2673,"ogSiteName":685,"ogType":686,"canonicalUrls":2673,"schema":2674},"How-to: GitLab Single Sign-on with SAML, SCIM, and Azure’s Entra ID","Follow this detailed walk-through of the configuration steps required to configure GitLab Single Sign-on, using Microsoft Azure’s Entra ID as the identity provider.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098047/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_1097303277_6gTk7M1DNx0tFuovupVFB1_1750098046895.jpg","https://about.gitlab.com/blog/how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How-to: GitLab Single Sign-on with SAML, SCIM, and Azure’s Entra ID\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Rob Jackson\"}],\n        \"datePublished\": \"2025-01-23\",\n      }",{"title":2670,"description":2671,"authors":2676,"heroImage":2672,"date":2678,"body":2679,"category":830,"tags":2680},[2677],"Rob Jackson","2025-01-23","As organizations increase in size, it becomes increasingly difficult and critical to ensure that the right team members have access to the right groups and projects within their development platform. GitLab offers some powerful methods to manage user access, especially now with [custom roles](https://about.gitlab.com/blog/how-to-tailor-gitlab-access-with-custom-roles/), but performing this at scale through a point-and-click user interface can be frustrating. However, all is not lost. You can use Security Assertion Markup Language (SAML) and System for Cross-domain Identity Management (SCIM) as a solution. (There are moments where I’m grateful for acronyms.) \n\nI was researching this topic for a particular customer, and walking through the GitLab documentation on the capabilities, but I never felt like I truly understood the integration. As is often the case, especially when dealing with integrating components, the knowledge from experience far outweighs that gained from reading or watching. In that light, I wanted to share my steps along this path and invite you all to join me. All you need is a free trial of Microsoft Azure Entra ID and GitLab Premium with a top-level group on GitLab.com.  \n\n**Note:** This exercise produces a working integration, however, for production environments there may be necessary deviations. For example, the user account email for the identity provider (Entra ID in this case) will likely not match your GitLab account email. \n\n## Creating the application in Entra ID\n\nFirst, go to the Entra ID admin center. Within the **Applications** area, select **Enterprise Applications**. We’re going to create a new application, and then create our own application.\n\n![Entra ID application creation flow](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image13_aHR0cHM6_1750098073325.png)\n\n\u003Ccenter>\u003Ci>Figure 1: Entra ID application creation flow\u003C/i>\u003C/center>\u003Cbr>\n\nWith our new application created, we can start configuring the single sign-on (SSO) parameters for our application. For this task, you may want to have side-by-side browser windows. One window on your Entra ID application, and another window on the SAML settings for your GitLab group. Those settings are located under **Settings**, then SAML SSO on the left side of your GitLab window, as shown in Figure 2. If you don’t see this option, you aren’t in the top-level group, don’t have permission to configure SAML, or don’t have GitLab Premium enabled for that group.\n\n![GitLab SAML configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098073326.png)\n\n\u003Ccenter>\u003Ci>Figure 2: GitLab SAML configuration\u003C/i>\u003C/center>\u003Cbr>\n\nWithin your Entra ID interface, select **Single sign-on** and click the SAML card.\n\n![Entra ID SAML configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image24_aHR0cHM6_1750098073328.png)\n\n\u003Ccenter>\u003Ci>Figure 3: Entra ID SAML configuration\u003C/i>\u003C/center>\u003Cbr>\n\nWith the side-by-side view, the SAML configuration settings are on the left and the GitLab SSO settings on the right. \n\n![Side-by-side view of Entra ID and GitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image16_aHR0cHM6_1750098073330.png)\n\n\u003Ccenter>\u003Ci>Figure 4: Side-by-side view of Entra ID and GitLab\u003C/i>\u003C/center>\u003Cbr>\n\nNow we can start copying and pasting parameters. Within the Entra ID interface, select **Edit** within the “Basic SAML Configuration” block. The parameter sources and destination are identified in the following table.\n\n| Source (GitLab) | Destination (Entra ID) |\n| :---------------- | :------: | \n| Identifier        |   Identifier (Entity ID)   | \n| Assertion consumer service URL |   Reply URL (Assertion Consumer Service URL)   | \n| GitLab single sign-on URL    |  Sign on URL (Optional)   | \n\n\u003Cbr>\nOnce completed, your side-by-side view should appear similar to the following (noting the URLs are unique to your environment).\u003Cbr>\n\n![Completed basic SAML SSO configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098073332.png)\n\n\u003Ccenter>\u003Ci>Figure 5: Completed basic SAML SSO configuration\u003C/i>\u003C/center>\u003Cbr>\n\nClick **Save** within the Entra ID “Basic SML Configuration” window to save your hard work thus far. Note: You may need to click on the “X” in the upper right of the “Basic SAML Configuration” window if it doesn’t close automatically. \n\nAfter this window closes, you may get a popup to test single sign-on with your application. Select **No, I’ll test later**, because we still have more work to do (there is always more work to do). \n\n## Configuring attributes and claims\n\nWithin the Entra ID user interface, look for the section for “Attributes and Claims,” and click the **Edit** pencil icon. The first thing we want to do is modify the Unique User identifier (Name ID) value, so click on that row and set the Source attribute to **user.objectid**. Additionally, the Name identifier format must be updated, and set to **Persistent**.\n\n![Configuring attributes and claims](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image14_aHR0cHM6_1750098073333.png)\n\n\u003Ccenter>\u003Ci>Figure 6: Configuring attributes and claims\u003C/i>\u003C/center>\u003Cbr>\n\nSave that claim configuration. Now we have additional claims to configure, but there are only three that we need here. So, feel free to go wild and delete those default four items under **Additional claims**, or you can edit the existing ones to match the table below. Note that these values (specifically, the Name) are case sensitive. \n\u003Cbr>\n\n| Name | Namespace | Source Attribute |\n| :---------------- | :------: | :------: | \n|emailaddress |http://schemas.microsoft.com/ws/2008/06/identity/claims | user.otheremail |\n| NameID | http://schemas.microsoft.com/ws/2008/06/identity/claims |user.objectid |\n\n\u003Cbr>\n\nThe resulting claims configuration should appear as follows. Note the use of **otheremail** for the “emailaddress” attribute. This was necessary for me as my primary email addresses within Entra ID are not the addresses used on GitLab.com. If you recall, when I set up my “user,\" I modified the contact information to include my gitlab.com email address as one of my “Other emails.” \n\n![Configuring the claims](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image21_aHR0cHM6_1750098073335.png)  \n\n\u003Ccenter>\u003Ci>Figure 7: Configuring the claims\u003C/i>\u003C/center>\u003Cbr>\n\nWith your attributes configured, under the Advance settings, enable **Include attribute name format** setting.\n\n![Advanced claims configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098073336.png)\n\n\u003Ccenter>\u003Ci>Figure 8: Advanced claims configuration\u003C/i>\u003C/center>\u003Cbr>\n\nYour \"Attributes and Claims\" window should now look similar to Figure 9 below.\n\n![Configured attributes and claims](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image18_aHR0cHM6_1750098073337.png)\n\n\u003Ccenter>\u003Ci>Figure 9: Configured attributes and claims\u003C/i>\u003C/center>\u003Cbr>\n\nIf you’re happy, or at least relatively content, with your configuration, click the “X” in the top right corner of the \"Attributes and Claims\" window to close it. \n\n## Configuring and assigning users\n\nNow that we have our application configured, we need to ensure that our users have been assigned to that application. I'll assume you’re working with a test instance that does not have the same email address as what is configured within your GitLab.com namespace. \n\nSo let’s go to the “Users and groups” within the Entra ID user interface for your configured application.\n\n![Managing application users and groups](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image17_aHR0cHM6_1750098073338.png)\n\n\u003Ccenter>\u003Ci>Figure 10: Managing application users and groups\u003C/i>\u003C/center>\u003Cbr>\n\nSelect **Add user/group**, and under the “Users and groups” where it says “None Selected,” click that text. Now you can select the user(s) to add to your application. These are the users that will be permitted to log into GitLab, authenticating themselves through Entra ID.\n\n![User selection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image23_aHR0cHM6_1750098073339.png)\n\n\u003Ccenter>\u003Ci>Figure 11: User selection\u003C/i>\u003C/center>\u003Cbr>\n\nOnce selected, at the bottom of that page, click **Select**, and at the bottom of the next, select **Assign**. Now you should have a user assigned to your application.\n\n![User assigned to application](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image12_aHR0cHM6_1750098073340.png)\n\n\u003Ccenter>\u003Ci>Figure 12: User assigned to application\u003C/i>\u003C/center>\u003Cbr>\n\nNext, we need to ensure that the GitLab.com email address for that user is configured correctly. By clicking on the user itself, we can modify or configure some  additional information about that user. We can see below, the User principal name, which is based on an “onmicrosoft” domain. This is not the email address I have associated with my GitLab.com account. If you recall that we set the “Email address” attribute to “otheremail,” this is where we now configure that “other” email address. \n\n![User properties](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image20_aHR0cHM6_1750098073341.png)\n\n\u003Ccenter>\u003Ci>Figure 13: User properties\u003C/i>\u003C/center>\u003Cbr>\n\nClick the option to **Edit properties** for the user, and click on the **Contact Information** heading. Here we can add other emails – more specifically, the email address utilized for your GitLab.com account.  \n\n![Configuration of alternate email address](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image15_aHR0cHM6_1750098073342.png)\n\n\u003Ccenter>\u003Ci>Figure 14: Configuration of alternate email address\u003C/i>\u003C/center>\u003Cbr>\n\nThat should complete the configuration parameters that we need in Entra ID, but wait, there’s more. \n\nWithin the GitLab side now, you will need to configure a couple parameters. First, you might as well enable SAML for the group as that’s kind of a key piece here. GitLab offers some additional options to disable password authentication or enforce SSO to reduce the security risks within your application, but we’ll leave those unchecked for now. Similar to the table above, we’ll need a couple things from Entra ID to configure into GitLab. Please refer to the table below. \n\u003Cbr>\n\n| Source (Entra ID) | Destination (GitLab) | \n| :---------------- | :------: | \n|Login URL |Identity provider single sign-on URL |\n| Thumbprint | Certificate fingerprint|\n\n\u003Cbr>\n\n![GitLab SAML configuration from Entra ID](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image25_aHR0cHM6_1750098073343.png)\n\n\u003Ccenter>\u003Ci>Figure 15: GitLab SAML configuration from Entra ID\u003C/i>\u003C/center>\u003Cbr>\n\nLastly, you want to configure the default membership role for users logging in via SAML. Note that the access that you set for users here will cascade down to other groups and projects within your top-level group. Therefore, I would strongly recommend NOT setting this role to be “Owner.” Either “Guest” or “Minimal Access” would be acceptable options here, depending on the security posture of your organization. For more information about what these roles can and can not do, refer to the GitLab documentation on [Roles and Permissions](https://docs.gitlab.com/ee/user/permissions.html#roles). Now, save your work on the GitLab interface by clicking that beautiful blue **Save changes** button.\n\nWith your GitLab settings saved, you can now test your setup. I would encourage you to do this both through the “Verify SAML Configuration” on the GitLab system as well as with the Entra ID SSO \"Test\" button.\n\n## Troubleshooting SAML\n\nIn addition to the troubleshooting steps included within [GitLab documentation](https://docs.gitlab.com/ee/user/group/saml_sso/troubleshooting.html), I wanted to include a couple other items that I personally experienced. \n\nIf you get an error stating that the SAML reference did not contain an email address, check the Claim name for your email within the “Attributes and Claims” section within your Entra ID application. With GitLab 16.7, we added support for the “2008” attribute names, and at least for the email address setting, I found the default “xmlsoap” name for the email address claim to be a disappointing failure. \n\nAnother common error is “SAML Name ID and email address do not match your user account.” As you may suspect, this error is caused by a mismatch of the “NameID” and “emailaddress” attributes within the Entra ID application. This could be a misconfiguration of the “Attributes and Claims,” but it could also be that the properties of your test user don’t match your configuration. One helpful method to identify exactly what is coming through the SAML exchange is to use a SAML Tracer or SAML Message Decoder plugin with your web browser. \n\n## SCIM\n\nNow that you have SAML configured to enable users to log in via your Entra ID application, let’s make sure that people are assigned to the proper group(s) upon login. This can be incredibly helpful at scale, where instead of manually identifying which groups the particular users belong to, GitLab can learn this information from your identity application, Entra ID in this case. \n\nBecause SCIM utilizes groups to identify group membership, we need to create a group within Entra ID and add the relevant user(s) to the group. For this we’ll need the main administration menu for Entra ID. \n\n![Entra ID Group configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image19_aHR0cHM6_1750098073344.png)\n\n\u003Ccenter>\u003Ci>Figure 16: Entra ID Group configuration\u003C/i>\u003C/center>\u003Cbr>\n\nWe’re going to create a new group and assign our user(s) to that group. So click **New group** and configure a new group, which only requires you to configure a “Group name.” I used the default group type of “Security.” Leave the “Membership type” as “Assigned.” From this window, we can also assign the members.\n\n![Creating a New Entra ID Group](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098073345.png)\n\n\u003Ccenter>\u003Ci>Figure 17: Creating a New Entra ID Group\u003C/i>\u003C/center>\u003Cbr>\n\nOnce you’ve added the member(s), click **Create** in the bottom of that window. With your group created, and the user(s) assigned to the group, we can configure SCIM.\n\nImmediately below the SAML configuration section within the GitLab UI, you’ll see the “SCIM Token” area. Here you can generate a new token, and copy the endpoint URL, both of which will be useful for the next steps. Note that if you forget or already have a SCIM token, it can be reset. \n\n![SCIM token and endpoint within GitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098073345.png)\n\n\u003Ccenter>\u003Ci>Figure 18: SCIM token and endpoint within GitLab\u003C/i>\u003C/center>\u003Cbr>\n\nWith this information saved, return to your Entra ID application configuration. Within the left side menu, you’ll find the following: \n\n![Provisioning SCIM within Entra ID](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098073346.png)\n\n\u003Ccenter>\u003Ci>Figure 19: Provisioning SCIM within Entra ID\u003C/i>\u003C/center>\u003Cbr>\n\nWithin the \"Provisioning\" section, click on **New Configuration**, which opens a new page where that token and URL from GitLab will be used. \n\n![New provisioning configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image22_aHR0cHM6_1750098073348.png)\n\n\u003Ccenter>\u003Ci>Figure 20: New provisioning configuration\u003C/i>\u003C/center>\u003Cbr>\n\nFeel free to test the connection to ensure that you’ve configured the parameters properly. After testing, click on the **Create** button to establish the configuration and work on our mappings and settings. You may need to click the “X” in the top right corner of the panel to return to the overview configuration.\n\nExpand the “Mappings,” which includes two parameters; “Provision Microsoft Entra ID Groups” and “Provision Microsoft Entra ID Users.” SCIM group provisioning isn’t currently supported in GitLab, and although it doesn’t break the integration, keeping group provisioning enabled may cause negligible error messages. Therefore, we want to disable “Provision Microsoft Entra ID Groups,” so click that entry and set the “Enabled” field to “No.” \n\n![Provisioning attribute mapping](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098073349.png)\n\n\u003Ccenter>\u003Ci>Figure 21: Provisioning attribute mapping\u003C/i>\u003C/center>\u003Cbr>\n\nSave that configuration and select “Provision Microsoft Entra ID Users.” Validate that all three \"Target Object Actions\" are enabled, and then proceed to the “Attribute Mapping” section. Delete all existing mappings available to delete (I find this easier because attributes can’t be assigned twice), and then configure the Attribute Mappings per the following table:\n\n| customappsso Attribute (Destination) | Microsoft Entra ID Attribute (Source) | Matching Precedence | Mapping Type |\n| :---------------- | :------: | :------: | :------: | \n|externalID|objectId|1|Direct|\n|active|Switch([IsSoftDeleted], , \"False\", \"True\", \"True\", \"False\")| |Expression|\n|userName|mailNickname| |Direct|\n|name.formatted|displayName| |Direct|\n|Emails[type eq “other”].value|userPrincipalName||Direct|\n\n\u003Cbr>\n\n![Editing attributes](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image26_aHR0cHM6_1750098073349.png)\n\n\u003Ccenter>\u003Ci>Figure 22: Editing attributes\u003C/i>\u003C/center>\u003Cbr>\n\nAfter configuring all of the attribute mappings, the result should be similar to that found in Figure 22.\n\n![Completed attribute mapping configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098073350.png)\n\n\u003Ccenter>\u003Ci>Figure 23: Completed attribute mapping configuration\u003C/i>\u003C/center>\u003Cbr>\n\nNote the use of the “other” email within the **customappssso** attribute. This relates back to the “other” email we configured for the user back in the Entra ID user properties. In a production situation, the emails for the SSO account and the email address for the account within GitLab should match. \n\nWith your mapping complete (congratulations, Ptolemy), there are some advanced configuration settings necessary. Underneath the \"Attribute Mappings,\" click the box for “Show advanced options.” Once this box is checked, a link called “Edit attribute list for customappsso” is revealed.\n\n![Advanced attribute configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098073351.png)\n\n\u003Ccenter>\u003Ci>Figure 24: Advanced attribute configuration\u003C/i>\u003C/center>\u003Cbr>\n\nClick that link, and ensure that the Name “ID” is both “Primary Key” and “Required,” and that “externalID” is also “Required.” These attributes both refer to a unique user ID generated by Entra ID. However, although the “id” itself is required, it is not consistently provided within the API calls. Therefore, GitLab relies on the “externalID” to ensure the proper connection between the Entra ID and GitLab user accounts. \n\n![Required attribute list](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098073351.png)\n\n\u003Ccenter>\u003Ci>Figure 25: Required attribute list\u003C/i>\u003C/center>\u003Cbr>\n\nSave these settings, and then close the “Attribute Mapping” page with the “X” in the top right of the window. Return to the \"Application Provisioning\" section and click **Start provisioning**. \n\nWithin GitLab, we need to configure the association between the group we configured within Entra ID and the level of access we want those users to have within the GitLab top-level group. Note that this association can be configured on each sub-group within GitLab for more extensive provisioning, but within GitLab, permissions flow downhill. Whatever permission you set for a user at a top-level group, or sub-group, will cascade down to all projects and groups contained therein. \n\nWithin the \"Settings\" portion of the GitLab menu, select **SAML Group Links**. Here is where you’ll configure the group name and determine what access level, or role, members of the Entra ID Group will have within this particular GitLab Group.\n\n![GitLab SAML Group link](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image27_aHR0cHM6_1750098073352.png)\n\n\u003Ccenter>\u003Ci>Figure 26: GitLab SAML Group link\u003C/i>\u003C/center>\u003Cbr>\n\nAs shown in Figure 26, I’ve configured my membership to The Academy such that any users within the dev-security group from Entra ID  will be granted Developer access. Note that this is a slight variation of what a typical production environment would look like. In most instances, the user account within the identity provider (Entra ID, in this case) would match the user’s corporate account email (and we wouldn’t require “other” emails). When configured properly, if the user does not already have an account on GitLab, one will be created for them tied to their SSO account. \n\n![GitLab SSO tutorial - image11](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098073/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750098073352.png)\n\n\u003Ccenter>\u003Ci>Figure 27: SAML Group Links configured\u003C/i>\u003C/center>\u003Cbr>\n\nNow that you’ve completed the configuration, give it a try! From another browser, preferably in private mode to ignore any cookies or other yummy artifacts, paste the link for the GitLab SSO URL found in the GitLab SAML configurations. You should be prompted to log in with your Entra ID credentials and gain the proper access to your GitLab group! \n\nCongratulations, you’ve made it! I hope you’ve learned from and appreciate the work here, and we can all rejoice in the fact that the users within the Play-Dough app can now all properly authenticate, with the right permissions, to The Academy!\n\n> Don't have a GitLab account? [Sign up for a free trial today](https://about.gitlab.com/free-trial/devsecops/).\n\n## Read more\n- [The ultimate guide to enabling SAML and SSO on GitLab.com](https://about.gitlab.com/blog/the-ultimate-guide-to-enabling-saml/)\n- [SAML SSO for GitLab.com groups documentation](https://docs.gitlab.com/ee/user/group/saml_sso/)",[9,830,495,851,1016],{"slug":2682,"featured":6,"template":700},"how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id","content:en-us:blog:how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id.yml","How To Gitlab Single Sign On With Saml Scim And Azures Entra Id","en-us/blog/how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id.yml","en-us/blog/how-to-gitlab-single-sign-on-with-saml-scim-and-azures-entra-id",{"_path":2688,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2689,"content":2695,"config":2701,"_id":2703,"_type":13,"title":2704,"_source":15,"_file":2705,"_stem":2706,"_extension":18},"/en-us/blog/how-to-harden-your-self-managed-gitlab-instance",{"title":2690,"description":2691,"ogTitle":2690,"ogDescription":2691,"noIndex":6,"ogImage":2692,"ogUrl":2693,"ogSiteName":685,"ogType":686,"canonicalUrls":2693,"schema":2694},"How to harden your self-managed GitLab instance","Learn seven easy steps to ensure your self-managed GitLab instance is as secure as possible.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664923/Blog/Hero%20Images/security-checklist.png","https://about.gitlab.com/blog/how-to-harden-your-self-managed-gitlab-instance","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to harden your self-managed GitLab instance\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ayoub Fandi\"}],\n        \"datePublished\": \"2023-05-23\",\n      }",{"title":2690,"description":2691,"authors":2696,"heroImage":2692,"date":2698,"body":2699,"category":785,"tags":2700},[2697],"Ayoub Fandi","2023-05-23","\n\"You are as secure as your weakest link\" is a well-understood phrase. If attackers find their way in, they will exploit any gaps in your security configurations. Hardening — the process of turning off unused features and making adjustments to settings that have security implications — is critical to limit your attack surface and reduce potential attack vectors.\n\nHardening ensures that your application (in this case GitLab) is as secure as it can be. The goal is simple: minimize risk while still preserving enough functionality for users to remain as productive as possible.\n\n## Guiding principles\nThese principles should guide the way you approach hardening. The security activities in the checklist below will tie back to one or another of these principles.\n\n### Layered security\nThe idea behind this is simple. If there are two ways to implement security, both ways should be implemented instead of just one. You can try to combine as many methods as possible. \n\nFor instance, if you are trying to secure access to your service, you could combine a complex password with hardware tokens and multifactor authentication. This approach is also called defense-in-depth.\n\n### No security through obscurity\nThe idea of hiding things works in many cases, but not so in the information security world. The premise that if something is hidden then it is more secure isn’t a viable approach today. \n\nCurrent scanning capabilities available to attackers shatter through obscurity. It is very easy for anyone to scan for open ports on a system. If you’ve swapped the SSH port TCP 22 to a different port, it would be picked up by a network scanning tool such as Nmap. \n\nAs GitLab is committed to transparency and open source, our approach is at odds with security through obscurity. The goal of security is to remove any security through obscurity. Our documentation is available to all and security best practices are clearly labeled and detailed.\n\n### Reducing the attack surface\nGitLab comprises numerous components, services, and dependencies. A critical aspect of security is: The more components you have, the more entryways attackers have.\n\nA good rule to keep in mind is to always disable services that you do not need to run the application. If there are features that aren’t used, disabling the related services will reduce the potential attack surface and make you more secure.\n\n## 7 steps to secure your self-managed instance\nLet’s go through seven easy steps to quickly harden your self-managed instance. These quick wins are great first steps towards securing your installation. Of course, refer to the [documentation](https://docs.gitlab.com/ee/security/) for additional details and further guidance on each section.\n\n### 1. Enable multi-factor authentication\n**Admin > Settings > General > Sign-in restrictions**\n\nEnsure that the checkbox next to _Two-factor authentication_ (2FA) is **checked**. The default setting for _Two-factor grace period_ is 48 hours. Adjust it to a lower value, such as **8 hours**.\n\nEnsure the checkbox next to _Enable admin mode_ is **checked** so that _Admin Mode_ is **active**. Users with Admin access will have to use additional authentication to perform administrative tasks. With 2FA enabled, this will require additional 2FA authentication by the user.\n\nFor more detailed information, refer to the documentation on [sign-in restrictions](https://docs.gitlab.com/ee/administration/settings/sign_in_restrictions.html).\n\n### 2. Enforce additional sign-up checks\n**Admin > Settings > General > Sign-up restrictions**\n\nNext to _Sign-up enabled_ ensure the checkbox is **unchecked**.\n\nUnder _Email confirmation settings_ ensure that **Hard** is selected. This will require the user to verify their email address during the sign-up process before their account is allowed access.\n\nThe _Minimum password length (number of characters)_ default setting of 12 characters is fine if additional authentication techniques are enforced. Options available for password complexity include _Require numbers_, _Require uppercase letters_, _Require lowercase letters_, and _Require symbols_. Check these boxes depending on your internal password standard (also check out [NIST SP 800-63B](https://pages.nist.gov/800-63-3/sp800-63b.html)).\n\nIf all users' email addresses are under a single domain (e.g., example.com), consider **adding it** to the _Allowed domains for sign-ups_. This will prevent those with email addresses associated with other domains from signing up.\nFor more detailed information, refer to the documentation on [sign-up restrictions](https://docs.gitlab.com/ee/administration/settings/sign_up_restrictions.html).\n\n### 3. Limit public visibility of your groups and projects\n**Admin > Settings > General > Visibility and access control**\n\nThe _Default project visibility_ and _Default group visibility_ for any newly created project or group should be set to **Private** by default. Only users that are granted specific access to a project or group will be able to access these resources. This can be adjusted later if necessary or when creating a new project or group. This ensures the default mode is secure to prevent accidental disclosure of information.\n\nFor more details on Visibility and access control [refer to the documentation](https://docs.gitlab.com/ee/administration/settings/visibility_and_access_controls.html).\n\n### 4. Harden your SSH settings\n**Admin > Settings > General > Visibility and access control**\n\nTypically, under _Enabled Git access protocols_ it will be set to _Both SSH and HTTP(S)_. If one of the Git protocols is not in use by your users, set it to **either** _Only SSH_ or _Only HTTP(S)_ accordingly. This will reduce the attack surface by limiting possibilities of compromise through an unused protocol.\nFor SSH key types, the most recommended algorithms to use are, in order: \n1. ED25519\n1. RSA \n1. ECDSA\n\nWhen configuring default types and lengths for SSH keys, keep in mind the list above.\n\nSpecific details on SSH settings can be found [here](https://docs.gitlab.com/ee/security/ssh_keys_restrictions.html) and [here](https://docs.gitlab.com/ee/administration/settings/visibility_and_access_controls.html#configure-enabled-git-access-protocols) for Git Access protocols.\n\n### 5. Review the account and limit settings\n**Admin > Settings > General > Account and limit settings**\n\nThis section allows you to limit the size of attachments, pushes, exports, imports, or repositories. As the specific size (in MB) will be tailored to your needs, review these settings and **set limits** in line with your internal policies.\n**Session duration for users** (in minutes) and **lifetime of SSH keys and all access tokens** (in days) can also be configured. Ensure the durations are in accordance with your internal policies and security best practices.\n\nReview the [documentation](https://docs.gitlab.com/ee/administration/settings/account_and_limit_settings.html) and apply changes that enforce your own policies.\n\n### 6. Secure your CI secrets\n**Admin > Settings > CI**\n\nPasswords, tokens, keys, and other secrets that require any level of protection should never be stored in plaintext. Instead, some type of **encrypted container technology (Secrets Manager)** should be implemented, such as GCP's Secret Manager, AWS Key Management Service (KMS), or HashiCorp Vault. For self-managed and standalone instances, HashiCorp Vault is **recommended**, and many GitLab features can take advantage of Vault and are well described in the [documentation](https://docs.gitlab.com/search/?query=vault).\n\nFor external communications, ensure any connectivity with external hosts in your CI/CD process is using encrypted channels. The use of TLS 1.2 or above is highly recommended and where possible mutual TLS will help things considerably.\nFor details on the use of external secrets for your CI/CD pipeline, check [here](https://docs.gitlab.com/ee/ci/secrets/) for actual examples and configuration guides.\n\n### 7. Protect your pipelines for all branches\n**Admin > Settings > CI**\n\nPipelines are a part of jobs that execute steps in stages to automate tasks on behalf of the users of a project. They are a central component of CI/CD.\nBy default, only the default branch gets a protected pipeline. Configure your other branches with the same level of security by following [these simple steps](https://docs.gitlab.com/ee/user/project/protected_branches.html#configure-a-protected-branch). This considerably hardens your pipelines.\n\nThe security features enabled by default on protected pipelines are listed in our [documentation](https://docs.gitlab.com/ee/ci/pipelines/#pipeline-security-on-protected-branches).\n\nOnce the pipeline has run, the code will be deployed in an environment. To limit interactions with that environment and to protect it from unauthorized users, you can set your key environments as protected.\n\nPrerequisites and full process are available in the [documentation](https://docs.gitlab.com/ee/ci/environments/protected_environments.html).\n\n### Learn more \nThis is a high-level overview of the different areas to focus on when hardening your self-managed GitLab instance. A blog post can’t include every single security recommendation. That’s why we maintain detailed [security documentation](https://docs.gitlab.com/ee/security/) on how to secure your installation. \n\nPlease refer to the documentation as the single source of truth on hardening. Hopefully, with the help of the action items highlighted above, you’ll harden your self-managed GitLab instance while preserving agility and speed.\n\nIf you want to learn more about how we do security **at GitLab**, review the [security section](https://about.gitlab.com/handbook/security/) of the handbook.\n",[830,9,851],{"slug":2702,"featured":6,"template":700},"how-to-harden-your-self-managed-gitlab-instance","content:en-us:blog:how-to-harden-your-self-managed-gitlab-instance.yml","How To Harden Your Self Managed Gitlab Instance","en-us/blog/how-to-harden-your-self-managed-gitlab-instance.yml","en-us/blog/how-to-harden-your-self-managed-gitlab-instance",{"_path":2708,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2709,"content":2715,"config":2722,"_id":2724,"_type":13,"title":2725,"_source":15,"_file":2726,"_stem":2727,"_extension":18},"/en-us/blog/how-to-harmonize-agile-sprints-with-product-roadmaps",{"title":2710,"description":2711,"ogTitle":2710,"ogDescription":2711,"noIndex":6,"ogImage":2712,"ogUrl":2713,"ogSiteName":685,"ogType":686,"canonicalUrls":2713,"schema":2714},"How to harmonize Agile sprints with product roadmaps","Apply best practices and GitLab features to your product journey, including creating centralized roadmaps, conducting review sessions, and tracking sprint lifecycles.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097231/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2821%29_2pdp2MNB7SoP4MhhiI1WIa_1750097230664.png","https://about.gitlab.com/blog/how-to-harmonize-agile-sprints-with-product-roadmaps","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to harmonize Agile sprints with product roadmaps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Amanda Rueda\"}],\n        \"datePublished\": \"2025-02-04\",\n      }",{"title":2710,"description":2711,"authors":2716,"heroImage":2712,"date":2718,"body":2719,"category":2720,"tags":2721},[2717],"Amanda Rueda","2025-02-04","Picture this: Product and Development teams are working in isolation. Product has created a 12-month roadmap and communicated it to internal stakeholders but didn't review it with their development team. Dev starts building the features planned for the upcoming sprint without considering the broader product roadmap, leading to missed opportunities to optimize timing, like running projects in parallel, accounting for team capacity, or building reusable APIs that could serve multiple initiatives. The lack of coordination results in inefficiencies and delayed value delivery.\n\nBalancing short-term wins with long-term vision isn’t easy; it requires clear communication, aligned priorities, and the right tools. In this guide, you'll learn strategies to help harmonize your Agile sprints with strategic roadmaps, tackle common challenges, and uncover actionable solutions tailored to your teams.\n\n## The importance of a single source of truth\n\nA consistent single source of truth for roadmaps with longer-range goals ensures you and your teams have access to up-to-date information about the bigger picture. In practice, this means maintaining a single, regularly updated platform where all roadmap details reside rather than keeping versions of the roadmap across multiple formats, each typically with slightly different information, causing a misaligned understanding of where you're headed.\n\n### Create a centralized roadmap\n\nBy creating a centralized roadmap for your team, you can:\n\n* communicate long-range strategy\n* minimize miscommunication\n* facilitate cross-functional alignment\n* quickly adapt to changes without losing context\n* self-serve information, reducing dependency on a single point of contact who retains the information\n\n***GitLab tip**: Use [epics](https://docs.gitlab.com/ee/user/group/epics/) and [Roadmap view](https://docs.gitlab.com/ee/user/group/roadmap/) to support both product planning and the transparent monitoring of delivery. The Roadmap view allows you to track progress, identify bottlenecks, and ensure alignment between high-level goals and sprint-level execution.*\n\n![Roadmap view for group](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097239/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097239117.png)\n\n## Collaborative roadmap review practices\n\nEstablish a regular review and sign-off process for roadmap updates that include Product, Engineering, and UX as part of the [product trio](https://www.producttalk.org/product-trio/). Collaborative reviews help you maintain alignment and minimize risk. At GitLab, I meet with my engineering manager and UX designer monthly to review and obtain sign-offs on any changes. We maintain a running sign-off on the roadmap wiki page itself that holds us accountable for keeping the schedule and provides transparency to the rest of the organization.\n\n#### How to extract value from review sessions\n\nTo make the most of the review session, aim for the following best practices:\n\n* Schedule routine reviews, monthly or quarterly, depending on how frequently the roadmap tends to fluctuate at your organization.\n* Validate alignment between product goals, UX lead time, and technical feasibility by discussing potential risks and dependencies upfront.\n  * Validate that the roadmap reflects current organizational business objectives.\n  * Ensure that design timelines are realistic and consider research or validation needs.\n  * Confirm that the roadmap allocates time for technical preparation, such as technical spikes or investigations, and ensures alignment with broader engineering priorities.\n* Optimize team utilization by considering capacity constraints and ensuring the sequence of work aligns with the team’s skill profile. This includes avoiding periods of underutilization or skill mismatches while effectively planning for situations like staffing level drops during holidays.\n* Right-size scope and set appropriate expectations about what can be achieved. We all want to do it all, but perfection is the enemy of progress so prioritize what truly matters to deliver incremental value efficiently. Seek opportunities to optimize by identifying ways to iterate or increase velocity, such as adjusting the order of work to reduce dependencies or leveraging reusable components to streamline development.\n* Encourage open dialogue about trade-offs and priorities to ensure all perspectives are considered. This collaborative approach helps identify creative solutions to challenges and builds consensus on the best path forward.\n\n***GitLab tip**: Use a [GitLab Wiki](https://docs.gitlab.com/ee/user/project/wiki/) page to complement the [Roadmap](https://docs.gitlab.com/ee/user/group/roadmap/) feature. In the wiki, you can include expanded context about your product roadmap, such as business rationale, links to user research, RICE scores, and details about dependencies or risks. Link directly to the roadmap for easy access, and leverage the upcoming discussion threads feature to encourage async collaboration and feedback from your team.*\n\n![PlanFlow product roadmap](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097239/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097239118.png)\n\n## Continuous direction validation and progress measurement\n\nThe goal of a product roadmap isn’t just to stay on track – it’s to deliver real value to your customers. To make space for sharing ongoing user feedback and behavioral data consider incorporating regular touchpoints across your product trio outside of sprint cycles. These sessions can be used to review insights, analyze trends, and ensure that the product roadmap continues to reflect the evolving needs of your users. By grounding roadmap updates using real user insights, you’re not only delivering on outcomes but also adapting to what really matters to your customers.\n\nThe value you ship might come in the form of improved usability, reduced technical debt, or entirely new capabilities. When the product trio is aligned on the roadmap vision, they’re also aligned on the outcomes you’re working to achieve.\n\nTo measure whether you’re on track to deliver those outcomes, you need to closely scope the intended results. Scope creep, like late user story additions, can delay your ability to ship value. Additionally, it’s important to identify work that was delivered but doesn’t align with the roadmap and understand why.\n\n### Sprint planning\n\nRemaining aligned with your product roadmap starts with thoughtful sprint planning. Here are some best practices to keep your team on track and focused on delivering value:\n\n* Clearly define, and narrowly scope, desired outcomes to ensure high confidence in delivery.\n* Identify potential late additions or adjustments that could delay delivery, and build in buffers to maintain focus.\n* Align on the sequence of work with your team to optimize for capacity, skill profiles, and reducing dependencies.\n* To maintain focus and improve confidence of delivering on time, avoid planning to 100% of the team’s capacity. Leave room (10%-20%) for unknowns or new discoveries that may surface during the sprint.\n\n### During the sprint\n\nStaying aligned with your roadmap during the sprint requires focus, communication, and constant evaluation. While delivering value is the goal, it’s equally important to ensure the work in progress aligns with the outcomes you’ve scoped and planned.\n\n* Continuously validate the work in progress against roadmap outcomes to ensure every sprint contributes to the bigger picture.\n* Encourage the team to regularly check if they’re still working toward the intended goals and outcomes.\n* Maintain open communication throughout the sprint. Use daily standups or async updates to surface risks, unplanned work, or dependencies early and adjust where necessary.\n* Be ruthless about protecting the sprint. While the urge to solve emerging problems is natural, unplanned work should be carefully evaluated to avoid derailing agreed-upon priorities.\n* Proactively manage scope creep. If new work surfaces mid-sprint, assess whether it aligns with the current roadmap outcome’s narrowly scoped focus. While additional ideas or features may align conceptually with the broader outcome, they may not fit into the immediate plan to deliver value as soon as possible. Document these suggestions and evaluate if they should be considered as part of future iterations or as a nice-to-have for the future, rather than introducing them into the current sprint and delaying agreed-upon priorities.\n\n### Sprint retros\n\nIn your sprint retrospectives, take time to reflect with your team on how well you are collectively progressing toward your desired outcomes. Questions to ask:\n\n* Did any unplanned work get introduced during the sprint that delayed your ability to deliver value? Identify why it happened and what adjustments can be made.\n* Did you deliver any work that deviated from the roadmap? Discuss what led to this and what you can learn for future planning.\n\nFrom sprint planning through retrospectives, staying focused on delivering tangible outcomes to users and stakeholders is a team responsibility. By aligning every step of the way, you ensure that your roadmap remains a clear guide for delivering value efficiently and consistently.\n\n***GitLab tip:** Use [burndown charts](https://docs.gitlab.com/ee/user/project/milestones/burndown_and_burnup_charts.html) to visualize progress and detect deviations early, helping your team stay focused on delivering outcomes.*\n\n![Burndown chart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097239/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097239120.png)\n\n## Delivering roadmap outcomes with confidence\n\nHarmonizing Agile sprints with strategic roadmaps requires intentionality, team buy-in, and the proper tools. By creating a roadmap single source of truth, fostering collaborative reviews, and measuring progress towards outcomes, you can align execution with vision. With GitLab’s robust planning features, teams can turn challenges into opportunities for innovation and growth.\n\nReady to align your sprints with your strategic roadmap? [Start a free trial of GitLab](https://about.gitlab.com/free-trial/) today and explore the tools that can help you deliver outcomes with confidence.\n\n## Learn more\n\n- [Agile planning content hub](https://about.gitlab.com/blog/categories/agile-planning/)\n- [GitLab’s new Planner role for Agile planning teams](https://about.gitlab.com/blog/introducing-gitlabs-new-planner-role-for-agile-planning-teams/)\n- [Get to know the GitLab Wiki for effective knowledge management](https://about.gitlab.com/blog/get-to-know-the-gitlab-wiki-for-effective-knowledge-management/)","agile-planning",[996,9,956,495],{"slug":2723,"featured":90,"template":700},"how-to-harmonize-agile-sprints-with-product-roadmaps","content:en-us:blog:how-to-harmonize-agile-sprints-with-product-roadmaps.yml","How To Harmonize Agile Sprints With Product Roadmaps","en-us/blog/how-to-harmonize-agile-sprints-with-product-roadmaps.yml","en-us/blog/how-to-harmonize-agile-sprints-with-product-roadmaps",{"_path":2729,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2730,"content":2736,"config":2741,"_id":2743,"_type":13,"title":2744,"_source":15,"_file":2745,"_stem":2746,"_extension":18},"/en-us/blog/how-to-integrate-custom-security-scanners-into-gitlab",{"title":2731,"description":2732,"ogTitle":2731,"ogDescription":2732,"noIndex":6,"ogImage":2733,"ogUrl":2734,"ogSiteName":685,"ogType":686,"canonicalUrls":2734,"schema":2735},"How to integrate custom security scanners into GitLab","Learn how to extend the DevSecOps platform by adding custom security scanners to your workflows (includes an easy-to-follow tutorial).","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097082/Blog/Hero%20Images/Blog/Hero%20Images/securitycheck_securitycheck.png_1750097081856.png","https://about.gitlab.com/blog/how-to-integrate-custom-security-scanners-into-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to integrate custom security scanners into GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2024-02-27\",\n      }",{"title":2731,"description":2732,"authors":2737,"heroImage":2733,"date":2738,"body":2739,"category":830,"tags":2740},[1444],"2024-02-27","GitLab, the most comprehensive DevSecOps platform, has everything you need to plan, manage, build, deploy, secure, govern, and monitor your applications. However, there are instances where you may want to extend GitLab with third-party or custom tools. For example, you might need to migrate to a DevSecOps platform from separate solutions, evaluate third-party tools, or integrate proprietary or custom-built solutions into GitLab.\n\nHere's what is covered:\n- [GitLab DevSecOps platform extensibility](#gitlab-devsecops-platform-extensibility)\n- [GitLab security scanner integration](#gitlab-security-scanner-integration)\n  - [Merge request security widget](#merge-request-security-widget)\n  - [Pipeline Security section](#pipeline-security-section)\n  - [Vulnerability Report](#vulnerability-report)\n  - [Vulnerability pages](#vulnerability-pages)\n  - [Security dashboard](#security-dashboard)\n  - [Scan Result Policy integration](#scan-result-policy-integration)\n- [Tutorial: Integrating custom security scanners](#tutorial-integrating-custom-security-scanners)\n  - [Creating a custom security scanner](#creating-a-custom-security-scanner)\n  - [Integrating a custom security scanner with GitLab](#integrating-a-custom-security-scanner-with-gitlab)\n\n## GitLab DevSecOps platform extensibility\n\nGitLab can be extended in many ways to support enhanced functionality that your organization may require. Some common examples of these integrations include:\n\n- external application integrations such as Jenkins and Slack\n- external issue tracking integrations such as Bugzilla and Jira\n- external authentication provider integrations such as LDAP and SAML\n- external security scanner integrations such as Fortify and Checkmarx\n- ability to respond to leaked secrets such as AWS and GCP access keys\n\nYou can see all the available integrations in the [Integrate with GitLab documentation](https://docs.gitlab.com/ee/integration/). (Note: Not all integrations are listed in the documentation.)\n\n## GitLab security scanner integration\n\n[Third-party security scanners](https://docs.gitlab.com/ee/integration/#security-improvements) or [custom-built security scanners](https://gitlab.com/gitlab-de/tutorials/security-and-governance/custom-scanner-integration) can be integrated into GitLab to populate the merge request widget, Pipeline Security section, Vulnerability Report, vulnerability pages, Security dashboard, and Scan Result Policies. Let's review each integration.\n\n### Merge request security widget\n\nA merge request contains a security widget which displays a summary of the newly detected vulnerabilities.\n\n![integrating security scanners - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097088837.png)\n\n\u003Ccenter>\u003Ci>Merge request security widget\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nWhen you click on a vulnerability, you will see a popup that contains the following information:\n- status\n- description\n- project\n- file\n- identifiers\n- severity\n- tool\n- scanner provider\n\n![integrating security scanners - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097088838.png)\n\n\u003Ccenter>\u003Ci>Actionable vulnerabilities with details\u003C/i>\u003C/center>\n\n\u003Cp>\u003C/p>\n\nThese vulnerabilities are also actionable, which means that they can either be dismissed or a confidential issue can be created.\n\nThe results of a custom scanner can be used to populate the security widget. The vulnerability data is populated from the JSON schema the scanner emits.\n\n### Pipeline Security section\n\nAll enabled security analyzers run in the pipeline and output their results as artifacts. These artifacts are processed, including deduplication, and the results are listed on the Pipeline Security tab. From here, you can also download the resulting JSON files.\n\n![integrating security scanners - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750097088840.png)\n\n\u003Ccenter>\u003Ci>Pipeline Security tab\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nThe results of a custom scanner can be used to populate the Pipeline Security tab. The columns are filled in using the JSON schema the scanner emits.\n\n### Vulnerability Report\n\nThe Vulnerability Report provides information about vulnerabilities from scans of the default branch, including:\n\n- totals of vulnerabilities per severity level\n- filters for common vulnerability attributes\n- details of each vulnerability, presented in tabular layout\n\n![integrating security scanners - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097088842.png)\n\n\u003Ccenter>\u003Ci>Vulnerability Report\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nThe results of a custom scanner on the default branch can be used to populate the Vulnerability Report.\n\n### Vulnerability pages\n\nClicking on a vulnerability present within the Vulnerability Report takes you to its vulnerability page. Each vulnerability in a project has a vulnerability page that provides details such as:\n\n- description\n- when it was detected\n- current status\n- location detected\n- available actions\n- linked issues\n- actions log\n- solutions\n- identifier\n- training\n\nYou can use the data provided in the vulnerability page to triage a detected vulnerability as well as assist in its remediation.\n\n![integrating security scanners - image 5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097088844.png)\n\n\u003Ccenter>\u003Ci>Vulnerability page for secret detection vulnerability\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nThe results of a custom scanner can be used to populate the vulnerability page. The vulnerability data is populated from the JSON schema the scanner emits.\n\n### Security dashboard\n\nSecurity dashboards are used to assess the security posture of your applications. GitLab provides you with a collection of metrics, ratings, and charts for the vulnerabilities detected by the security scanners run on your project. The security dashboard provides data such as:\n\n- vulnerability trends over a 30-, 60-, or 90-day timeframe for all projects in a group\n- a letter grade rating for each project based on vulnerability severity\n- the total number of vulnerabilities detected within the last 365 days and their severity levels\n\n![integrating security scanners - image 6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097088846.png)\n\n\u003Ccenter>\u003Ci>Group-level Security dashboard\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nFrom the group-level Security dashboard you can click on a project to access its specific Security dashboard, which provides the 365-day view.\n\n![integrating security scanners - image 7](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097088847.png)\n\n\u003Ccenter>\u003Ci>Project-level Security dashboard\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\n### Scan Result Policy integration\n\nScan Result Policies are used to require approval based on the findings of one or more security scan jobs. This can prevent insecure code from being merged to production. Scan Result Policies are evaluated after a CI scanning job is fully executed, where policies are evaluated based on the job artifact reports that are published in the completed pipeline.\n\nFor example, you can create a Scan Result Policy that requires approval from project maintainers if a secret detection scanner finds any vulnerabilities. Here's how:\n\n1. On the left sidebar, select **Search or go to** and search for the project you wish to add a policy to.\n2. On the project left sidebar, go to **Secure > Policies**\n3. Select **New policy**\n4. In the **Scan result policy** section, select **Select policy**.\n5. Complete the fields:\n- Name: The name of the Policy\n- Description: The description of the Policy\n- Policy status: Whether it is enabled or not\n- Rules: The conditions that must be met for an action (require approval) to take place\n\n![integrating security scanners - image 8](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097088849.png)\n\u003Ccenter>\u003Ci>Scan Result Policy rules\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\n- Actions: The action to be taken whenever the conditions in the rules (defined vulnerabilities/licenses detected) are met\n\n![integrating security scanners - image 9](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750097088850.png)\n\n\u003Ccenter>\u003Ci>Scan Result Policy actions\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\n- Override project approval settings: If selected, the following choices will overwrite project settings but only affect the branches selected in the policy\n\n![integrating security scanners - image 11](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097088851.png)\n\n \u003Ccenter>\u003Ci>Scan Result Policy approval settings\u003C/i>\u003C/center>\n \u003Cp>\u003C/p>\n\n6. Press the \"Configure with a merge request\" button.\n\nOnce the Scan Result Policy has been merged, whenever you create a merge request and the criteria defined in the rules are met, then the defined action will be triggered. In this case, at least one approval will be required from a maintainer before the code can be merged.\n\n![integrated security scanner - image 10](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097089/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750097088852.png)\n\n\u003Ccenter>\u003Ci>Blocked merge request due to detected vulnerabilities\u003C/i>\u003C/center>\n\u003Cp>\u003C/p>\n\nThe results of a custom scanner can be fully integrated with Scan Result Policies. If the custom scanner detects a vulnerability, then approval will be required before the code can be merged. The scanner you select in a Scan Result Policy must be leveraging the appropriate JSON schema.\n\n## Tutorial: Integrating custom security scanners\n\nNow let’s get to the fun part – integrating a custom security scanner. In this tutorial, you will learn how to create a custom security scanner, as well as how to integrate it with GitLab. We will be leveraging the following projects:\n\n- [Fern Pattern Scanner](https://gitlab.com/gitlab-de/tutorials/security-and-governance/custom-scanner-integration/fern-pattern-scanner): Scans your files looking for specific patterns such as passwords, private keys, and social security numbers.\n- [Secret list](https://gitlab.com/gitlab-de/tutorials/security-and-governance/custom-scanner-integration/secret-list): Contains a list of user passwords, clients, and keys. This project is used to showcase how a custom security scanner can be integrated into GitLab.\n\nYou can watch the following video to see how the application was created and how it is used in detail:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/timMbl5SP-w?si=R2DKtZ5MmBR1rQFL\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Creating a custom security scanner\n\nNow let’s create a custom scanner that can be integrated into GitLab. Before a custom scanner can be fully integrated with GitLab, the scanner must:\n- scan a directory for defined patterns\n- emit a JSON following the appropriate schema\n- be containerized and accessible\n- provide a template to allow it to be run on another project\n\nWhen the [Fern Pattern scanner](https://gitlab.com/gitlab-de/tutorials/security-and-governance/custom-scanner-integration/fern-pattern-scanner) is run on a project using the provided template, it performs the following steps:\n1. Loads a set of rules which define patterns (regex) to detect.\n- Allow rules to be configurable to meet the changing needs of your organization.\n2. Scans files for defined patterns.\n3. Emits a JSON report following the Secret Detection schema.\n- Go templates are used in this project to create a JSON.\n- Depending on what your scanner will look for, make sure you use the appropriate schema.\n\nOnce the JSON report is loaded as an artifact into GitLab, it will populate the merge request widget, Vulnerability Report, vulnerability pages, Scan Result Policies, and Security dashboards as defined above.\n\n### Integrating a custom security scanner with GitLab\n\nOnce you have created your custom scanner that meets all the needs for integration, you can run it on GitLab.\n\nRunning a custom scanner is as easy as adding a template. We can see how the Fern Pattern scanner template is loaded by examining the `.gitlab-ci.yml` in the [Secret List](https://gitlab.com/gitlab-da/tutorials/security-and-governance/custom-scanner-integration/secret-list) project.\n\n1. Create a [.gitlab-ci.yml file](https://docs.gitlab.com/ee/ci/quick_start/#create-a-gitlab-ciyml-file) in the project you want the scanner to run on.\n2. Include the [Custom Scanner template](https://docs.gitlab.com/ee/ci/yaml/includes.html).\n    - You should also be able to configure the template with environment variables.\n3. Commit the file to the main branch.\n\nOnce the file has been committed, you can see that the custom scanner will run in your pipeline. Once the pipeline is complete, the scanner will populate all the areas defined above in the [GitLab security scanner integration](#gitlab-security-scanner-integration) section.\n\n## Read more\n\nCheck out these resources to learn more about GitLab and the other ways you can extend your DevSecOps platform:\n\n- [Security Scanner GitLab Integration](https://docs.gitlab.com/ee/development/integrations/secure.html)\n- [GitLab Partner Integrations](https://docs.gitlab.com/ee/integration/)\n- [Custom Security Scanner Projects Group](https://gitlab.com/gitlab-de/tutorials/security-and-governance/custom-scanner-integration)\n- [Automatic Response to a Secret Leak](https://docs.gitlab.com/ee/user/application_security/secret_detection/automatic_response.html)\n",[9,830,1057,495],{"slug":2742,"featured":90,"template":700},"how-to-integrate-custom-security-scanners-into-gitlab","content:en-us:blog:how-to-integrate-custom-security-scanners-into-gitlab.yml","How To Integrate Custom Security Scanners Into Gitlab","en-us/blog/how-to-integrate-custom-security-scanners-into-gitlab.yml","en-us/blog/how-to-integrate-custom-security-scanners-into-gitlab",{"_path":2748,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2749,"content":2755,"config":2761,"_id":2763,"_type":13,"title":2764,"_source":15,"_file":2765,"_stem":2766,"_extension":18},"/en-us/blog/how-to-leverage-gitlab-duo-for-enhanced-security-reporting",{"title":2750,"description":2751,"ogTitle":2750,"ogDescription":2751,"noIndex":6,"ogImage":2752,"ogUrl":2753,"ogSiteName":685,"ogType":686,"canonicalUrls":2753,"schema":2754},"How to leverage GitLab Duo for enhanced security reporting","Learn how GitLab Duo enables efficient, real-world security reporting for development, operations, and security teams.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098339/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%285%29_1iy516k40hwBDChKcUJ2zb_1750098339103.png","https://about.gitlab.com/blog/how-to-leverage-gitlab-duo-for-enhanced-security-reporting","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to leverage GitLab Duo for enhanced security reporting\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Valentine Mairet\"},{\"@type\":\"Person\",\"name\":\"David O'Regan\"}],\n        \"datePublished\": \"2024-12-03\",\n      }",{"title":2750,"description":2751,"authors":2756,"heroImage":2752,"date":2758,"body":2759,"category":718,"tags":2760},[2757,2383],"Valentine Mairet","2024-12-03","Good security reporting is crucial to maintain a good security posture because it provides detailed insights into incidents. With this information, organizations can better understand vulnerabilities, improve defenses, and prevent similar threats in the future. At GitLab, the [Security division](https://handbook.gitlab.com/handbook/security/#division-structure) has created use cases for GitLab Duo to improve reporting capabilities and enhance operational efficiency. \n\n## GitLab Duo’s security capabilities\n\nThe GitLab Security division uses GitLab’s built-in [incidents](https://docs.gitlab.com/ee/operations/incident_management/incidents.html) to manage and report on security incidents. Incidents are handled, documented, and resolved in GitLab, enabling the use of AI-driven [GitLab Duo](https://about.gitlab.com/gitlab-duo/) as an assistant when performing security operations like incident response. \n\nParticularly in incident analysis and reporting, GitLab Duo is highly efficient and accurate at creating proper documentation and is a great “pair programmer” when solving security incidents.\n\n## GitLab Duo features for security reporting\n\nGitLab Duo offers many features that enhance security reporting:\n\n- **Root Cause Analysis:** GitLab Duo can explain vulnerabilities and understand the context of an incident issue, making it an excellent assistant for performing root cause analyses of security incidents.\n- **Vulnerability Explanation:** Provides detailed insights into identified vulnerabilities, including potential exploitation methods and remediation steps. This feature aids developers and security analysts in understanding and addressing security issues effectively.\n- **Vulnerability Resolution:** Assists in fixing vulnerabilities by generating merge requests that address the identified issues, streamlining the remediation process.\n- **Code Explanation:** Helps users comprehend specific code segments by offering clear explanations, which is particularly useful when dealing with complex or unfamiliar codebases.\n- **Test Generation:** Facilitates early bug detection by generating tests for selected code, ensuring that security vulnerabilities are identified and addressed promptly.\n- **Refactor Code:** Suggests improvements or refactoring for selected code to enhance its quality and maintainability, contributing to a more secure codebase.\n- **Fix Code:** Identifies and rectifies quality issues such as bugs or typos in the selected code, helping maintain a robust and secure codebase.\n\n## Practical use cases\n\nFor the purpose of demonstrating practical use cases, the Security Incident Response Team created a dummy incident with following limited information:\n\n![Incident report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098346/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098346297.png)\n\nSeveral comments were added as the team would normally proceed:\n\n![Comments added to report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098346/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098346297.png)\n\n### Incident reporting\n\nGitLab Duo is able to comprehensively keep track of all information inside an incident issue, including the issue description, comments, and labels. When handling security incidents, information often is all over the place and can change over time. It can easily get lost or overlooked. GitLab Duo is excellent at finding relevant information again to create accurate incident reports. \n\nNavigate to your incident issue and open GitLab Duo Chat. You can engineer your prompt so that GitLab Duo takes your exact reporting requirements into account such as what sections you need and how they should be filled out. Here is an example of the prompt we use at GitLab Security:\n\n> Required sections:\n> - Executive Summary - bottom-line-up-front that is adequate for an audience like senior leadership and CISO\n> - Mitigations & Remediations\n> - Scope & Impact (Environments, customers, team members)\n> - Cause\n> - Detection Capabilities\n>\n> Based on the issue and the required section, write the security incident report. Flag any missing information. \n\nBelow is GitLab Duo’s output that we can use to create our extensive incident report:\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752175975/Blog/zxiafech08ck9gqjubaz.png\" alt=\"root cause analysis output\" width=\"400\" height=\"1200\">\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752175985/Blog/xjc9coah4oi7rz0daw2b.png\" alt=\"root cause analysis output\" width=\"400\" height=\"1200\">\n\n### Root Cause Analysis\n\nGitLab Duo is able to explain vulnerabilities and can understand the context of an incident issue. These capabilities make GitLab Duo an excellent assistant to perform root cause analyses of security incidents. \n\nAs seen above, GitLab Duo can write its own section on the cause of an incident based entirely on the issue contents. We can dig into this further with the assistance of GitLab Duo to identify several candidates for root cause(s).  \n\nOur prompt is built as such:\n\n> “The root cause of the incident was the exposure of a personal access token in CI job logs. The token was leaked in a project that was not included in the current list of projects for token detection, indicating a gap in the security monitoring process.”\n> Name a few potential root causes for a PAT finding itself in CI job logs.\n\nGitLab Duo’s response is the following:\n\n![GitLab Duo Root Cause Analysis response](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098346/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098346298.png)\n\nGitLab Duo Root Cause Analysis can help security teams identify avenues worth investigating. It has a pretty good grasp on security concepts and how Root Cause Analysis is led in general, so it can be a great assistant for these situations where the exact root of the problem is unclear. \n\n> Learn more about [GitLab Duo Root Cause Analysis](https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd/).\n\n### On-demand updates\n\nSecurity incidents move fast and can be confusing, and information is constantly changing and shifting around. Higher severity incidents require a certain degree of thoroughness when it comes to reporting to senior leadership, especially when the audience is at the Director+ level. Engineers who are in the midst of a security incident can find it difficlut to condense the information such that it is digestable for executives. \n\nWe saw above that GitLab Duo is capable of delivering a pretty good executive summary. When the incident is ongoing, we need to deliver regular updates to senior leadership on the incident status and next steps. GitLab Duo is a great help for that, as well. If information is scattered across the issue in the form of a description or comments, GitLab Duo can help reassemble this information into the “bottom-line-up-front,” or BLUF summary, we need for executive updates. \n\nWe’ve taken the same incident right before token revocation and asked GitLab Duo for a BLUF summary where the audience is the Director of Security Operations. \n\n![Executive Summary - GitLab Duo](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098346/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098346299.png)\n\n## Getting started with GitLab Duo for security\n\nGitLab Security has automated several parts of the reporting process with the help of GitLab Duo. But to get started, all you need is access to GitLab Duo Chat. GitLab Duo Chat can be your well-informed assistant for many security reporting cases and post-mortem analyses.\n\n## What’s next for GitLab Duo?\n\nGitLab is committed to continuously enhancing GitLab Duo’s capabilities. Future developments aim to integrate AI-driven features more deeply into the security workflow, providing proactive detection and resolution of vulnerabilities, streamlined incident management, and comprehensive reporting tools. These advancements will further empower security teams to maintain robust security postures and respond effectively to emerging threats.\n\n> [Try GitLab Duo for free](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/)!\n",[720,9,830,696,495],{"slug":2762,"featured":6,"template":700},"how-to-leverage-gitlab-duo-for-enhanced-security-reporting","content:en-us:blog:how-to-leverage-gitlab-duo-for-enhanced-security-reporting.yml","How To Leverage Gitlab Duo For Enhanced Security Reporting","en-us/blog/how-to-leverage-gitlab-duo-for-enhanced-security-reporting.yml","en-us/blog/how-to-leverage-gitlab-duo-for-enhanced-security-reporting",{"_path":2768,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2769,"content":2775,"config":2782,"_id":2784,"_type":13,"title":2785,"_source":15,"_file":2786,"_stem":2787,"_extension":18},"/en-us/blog/how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two",{"title":2770,"description":2771,"ogTitle":2770,"ogDescription":2771,"noIndex":6,"ogImage":2772,"ogUrl":2773,"ogSiteName":685,"ogType":686,"canonicalUrls":2773,"schema":2774},"Bamboo Server to GitLab CI migration: Advanced techniques","A real-world look at how a migrated CI/CD infrastructure will work in GitLab CI.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679081/Blog/Hero%20Images/jenkins-migration.jpg","https://about.gitlab.com/blog/how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to migrate Atlassian's Bamboo server's CI/CD infrastructure to GitLab CI, part two\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ivan Lychev\"}],\n        \"datePublished\": \"2022-07-11\",\n      }",{"title":2776,"description":2771,"authors":2777,"heroImage":2772,"date":2779,"body":2780,"category":785,"tags":2781},"How to migrate Atlassian's Bamboo server's CI/CD infrastructure to GitLab CI, part two",[2778],"Ivan Lychev","2022-07-11","\nIn [part one of our series](/blog/migration-from-atlassian-bamboo-server-to-gitlab-ci/), I showed you how to migrate from Atlassian’s Bamboo Server to GitLab CI/CD. In this blog post we’re going to take a deep dive into how it works from a user’s perspective.\n\n## Get started\n\nYou’ve deployed the demo so it’s time to play with it to understand how it works.\n\nLet's imagine that one of the members of our project is John Doe. He is a software engineer responsible for developing some components (app1, app2, and app3) of the entire product, and he and his team would like to test those components in several combinations in myriad preview environments. So, what does that look like?\n\nFirst of all, let’s make some commits to the app1, app2, and app3 source code and get successful builds upon those commits.\n\nAfter that, we should create releases for those apps to be able to deploy them (as the deployment part of the apps CI config only shows when being triggered by a Git tag, i.e., a GitLab release). A release can be created by launching the last step (`manual-create-release`) in a commit pipeline. That would give us a new release with the ugly name containing the date and commit SHA in the patch part (in accord to `semver` scheme):\n\n\n\n![app_gitlab_release](https://about.gitlab.com/images/blogimages/app_gitlab_release.png)\n\nOn the `Tags` tab for the same app you now can see a deployment part of the pipeline has been triggered by the just created GitLab release but no actual environments to deploy are displayed (the `_` item in the `Deploy-nonprod` stage is not an env):\n\n\n![absent_envs](https://about.gitlab.com/images/blogimages/absent_envs.png)\n\n\n## Create an environment\n\nBut before that we have to briefly switch to another team who is responsible for preparing infrastructure IaC templates. Navigate to the `infra/environment-blueprints` project and pretend you are a member of that team doing their job. Namely, imagine you have just created some initial set of IaC files (they are already kindly prepared by me and present in the repository). You’ve tested them and now you feel that they are ready to be used by the other members of the project. You indicate such a readiness of a particular version of the IaC files by giving it a GitTag. Let’s put a tag like `v1.0.0` onto the HEAD version.\n\nYou will see how the tags are going to be used immediately. But first let's make some changes to the IaC files (e.g., add a new resource for some of the apps) and create a second Git tag, let's say `v1.1.0`. So, at this moment we have two versions of IaC templates (or `blueprints`) for our infrastructure - `v1.0.0` and `v1.1.0`.\n\n## Deploy an app into the environment\n\nNow we can return back to John and his team. We assume John is somehow informed that the version of the IaC templates he should use is `v1.0.0`. He wants to create a new preview environment out of the IaC templates of that version and put app1 and app2 into that env. \n\n(Here starts a description of how a user interoperates with the `infrastructure-set` Git repo. Notice that though the eventual idea is that it should be a Merge Request workflow – where you first get a Terraform plan within a Merge Request and can apply such a plan by merging the MR – which is widely advocated by GitLab but for the sake of simplicity here the MR workflow is not implemented and instead direct push commits into a branch are made).\n\nJohn wants the env to be named `preview-for-johns-team`. He creates a new branch in the `infrastructure-set` repo with that name and puts two files into it: a `version.txt` containing text `v1.0.0` and `apps.txt` with text `app1 app2` inside (the files format and its content is utterly simplified). \n\nThe `infrastructure-set` pipeline is triggered by the new branch and first generates a Terraform plan using the set of the Terraform files indicated by the tag specified in `version.txt`. John reviews the plan and wants to proceed with creating the environment by starting the `Terraform-apply` stage:\n\n\n![new_env_pipeline](https://about.gitlab.com/images/blogimages/new_env_pipeline.png)\n\n\n(To store the Terraform plan as artifact and Terraform state the embedded features of GitLab are leveraged - [Package Registry](https://docs.gitlab.com/ee/user/packages/package_registry/) and [Terraform HTTP back-end by GitLab](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html).)\n\nNow return to the `app1` project and rerun the pipeline for the app1 release we created previously to make it regenerate a list of environments to deploy. You should see that the `preview-for-johns-team` item has appeared in the list of the environments:\n\n\n![new_env_in_the_deploy_pipeline](https://about.gitlab.com/images/blogimages/new_env_in_the_deploy_pipeline.png)\n\n\nClick the arrow button to deploy. Then refer to the `Deployments/Environments` section of the `app1` project to ensure a new env with the app1 release deployed into it is displayed.\n\nWe have successfully created a new environment and deployed one of the apps into it!\n\nNotice that although the above describes how users manually deploy the applications into an environment after it has been created which doesn’t look really convenient, in a real life scenario we most likely would have some additional step in the `infrastructure-set` pipeline that runs after Terraform successfully finishes creating an environment and triggers deployment pipelines for all the applications specified in the `apps.txt`. In that situation, we would need to establish which versions of the applications should be deployed in such an automated manner - for example, those might be the latest versions available for each app or the versions currently deployed to production, etc.\n\n## Update an environment's infrastructure\n\nJohn got notified that a new version of the infrastructure templates is available (you remember that `v1.1.0` tag in the `environment-blueprints` repo?). His team wants to assess how app1 would work within the new conditions. They decide to update an existing env, namely `preview-for-johns-team`, for that purpose. \n\nJohn walks to the `preview-for-johns-team` branch of the `environment-set` repo and changes `version.txt`'s content from `v1.0.0` to `v1.1.0`. The branch pipeline gets triggered and first shows John a Terraform plan for a diff comparing the current state of the environment. After reviewing and accepting that diff, John proceeds with actual updating the environment by launching `Terraform-apply` stage. That's it!\n\n## Advantages and disadvantages\n\n### Virtues\n\nGiven that this case assumes migrating from some existing CI/CD infrastructure based on Atlassian Bamboo with a lot of users who are familiar with it, the proposed solution leverages the native capabilities of GitLab so that it mostly keeps the concepts and workflows used with Bamboo. This strategy makes the process of migration more smooth for the users.\n\nThe solution sticks to the GitOps tenets and empowers a project with all the virtues provided by Git. For example, it's usually easy to track any changes in the infrastructure back to Git repos. (It may not be so easy for the `environment-set` project where we do not have the infrastructure changes captured in Git commits, but in that case a task of finding differences between two states of a particular environment can be accomplished by fetching the two versions of the `environment-blueprints` repo corresponding to those states denoted in the `version.txt` and figuring out the differences by using any apt tool.)\n\nThe solution tends to support user self-service where most of the tasks of changing the infrastructure can be performed only by those familiar with the basics of Git and Terraform. As a result, it offloads the DevOps team from some part of the work and removes dependence on the Ops department which comes in really handy, especially for large-scale projects.\n\n### Shortcomings\n\nBesides the mentioned deficits which stem from the necessity to utterly simplify all the aspects of this demo to make it comprehensible and possible to prepare in a sensible amount of time, this solution possesses some shortcomings that have to be resolved by using external tools to make this solution appropriate for a real life usage.\n\nFor example, there is no way to have a central dashboard with an aggregated view of all the environments with all the apps and their versions deployed into the envs. This would require creating some custom SPA web app which would gather information from GitLab via API.\n",[108,789,9],{"slug":2783,"featured":6,"template":700},"how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two","content:en-us:blog:how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two.yml","How To Migrate Atlassians Bamboo Servers Ci Cd Infrastructure To Gitlab Ci Part Two","en-us/blog/how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two.yml","en-us/blog/how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two",{"_path":2789,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2790,"content":2796,"config":2801,"_id":2803,"_type":13,"title":2804,"_source":15,"_file":2805,"_stem":2806,"_extension":18},"/en-us/blog/how-to-protect-gitlab-connected-ssh-key-with-yubikey",{"title":2791,"description":2792,"ogTitle":2791,"ogDescription":2792,"noIndex":6,"ogImage":2793,"ogUrl":2794,"ogSiteName":685,"ogType":686,"canonicalUrls":2794,"schema":2795},"How to protect GitLab-connected SSH key with Yubikey","Add a layer of security to SSH keys by restricting physical access to YubiKey.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667202/Blog/Hero%20Images/gitlabultimatesecurity.jpg","https://about.gitlab.com/blog/how-to-protect-gitlab-connected-ssh-key-with-yubikey","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to protect GitLab-connected SSH key with Yubikey\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Brendan O'Leary\"}],\n        \"datePublished\": \"2022-03-03\",\n      }",{"title":2791,"description":2792,"authors":2797,"heroImage":2793,"date":2798,"body":2799,"category":785,"tags":2800},[1237],"2022-03-03","\n[Two-factor authentication](https://docs.gitlab.com/ee/security/two_factor_authentication.html) is one of the best defenses we have as individuals for protecting our accounts and credentials. But not all 2FA methods are created equal. For example, SMS is vulnerable to [SIM-swapping](https://www.ic3.gov/Media/Y2022/PSA220208) attacks and thus doesn't always provide the extra security we would like.\n\nIdeally, everything I  want to connect to would use 2FA with dedicated 2FA hardware. With GitLab 14.8, you can now use 2FA hardware to protect your SSH keys, as I explain below.  \n\n## 2FA and SSH keys\n\nState-of-the-art 2FA uses a physical hardware device – often FIDO/U2F hardware – to verify your presence at the time of authentication. This provides two distinct factors as a means of authentication: something you know (your username and password, for instance) with something you have (the physical device). I have two [YubiKey](https://www.yubico.com/works-with-yubikey/catalog/gitlab/) devices that I use for this purpose – one that is always in a safe in my house and one that I generally keep with me and the computer I'm using to do work. And I have everything I can secure using this method, including my GitLab account.\n\nAnd that does a great job of securing my access to GitLab, the application front end, and the ability to create and modify API keys. But there is another way to authenticate to a git server: SSH keys. In this case, there's only one factor of authorization because the SSH key is on my computer. So you can imagine how excited I was to hear that GitLab added support for `ecdsa-sk` and `ed25519-sk` key types in [GitLab 14.8](/releases/2022/02/22/gitlab-14-8-released/#support-for-ecdsa-sk-and-ed25519-sk-ssh-keys).\n\n### What are `ecdsa-sk` and `ed25519-sk`?\n\nThese two new keys are close to the existing `ecdsa` (Elliptic Curve Digital Signature Algorithm) and `ed25519` (Edwards Curve Digital Signature Algorithm) keys already supported. But that `-sk` at the end adds the ability to verify the key with a FIDO/U2F device. \"SK\" here stands for \"security key\". [OpenSSH 8.2](https://www.openssh.com/txt/release-8.2) added this key type to the supported keys it can generate, interacting with the hardware device to authenticate user presence before allowing the key to be used.\n\nHowever, I still had a few things to do to be ready to use the new keys.\n\n## Updating OpenSSH \nMy daily driver computer is a 2021 iMac running macOS Big Sur version 11.6. When I ran to it to generate this new key, I encountered a problem. Supposedly my version of SSH didn't support `-sk` keys!\n\nNow, your mileage may vary here, but I was able to update the version of SSH my Mac uses by default by first running `brew install openssh`, which successfully installed OpenSSH 8.8. But when I ran `ssh -V` it still showed version 8.1. So how could I get the system to use the newly installed OpenSSH instead?\n\nThe easiest way I could think of to do that was to put the Homebrew version first in the $PATH variable. But where is that path? Luckily, I was able to find that (`/opt/homebrew/opt/openssh`) by running this command:\n\n`brew --prefix openssh`\n\nOnce I updated my $PATH variable to have that at the front, I got the desired outcome:\n\n```bash\n$  which ssh\n/opt/homebrew/opt/openssh/bin/ssh\n\n$ ssh -V\nOpenSSH_8.8p1, OpenSSL 1.1.1m  14 Dec 2021\n```\n\n## Generating the key\nNow that I was using the correct version of SSH, I was able to create my `ecdsa-sk` key by running: \n\n```bash\nssh-keygen -t ecdsa-sk -f ~/.ssh/id_ecdsa_sk\n```\n\nNow, the specific device I have only supports ECDSA and not EdDSA, which is why I went with `ecdsa-sk`. There also is an option to have the key reside ON the device itself (if supported by your hardware) with the `-O resident` flag like this:\n\n```bash\n$ ssh-keygen -t ecdsa-sk -O resident -f ~/.ssh/id_ecdsa_sk\n\nEnter PIN for authenticator:\nYou may need to touch your authenticator (again) to authorize key generation.\nEnter passphrase (empty for no passphrase):\nEnter same passphrase again:\nYour identification has been saved in /Users/brendan/.ssh/id_ecdsa_sk\nYour public key has been saved in /Users/brendan/.ssh/id_ecdsa_sk.pub\n```\n\nGenerating a resident key will make sharing this key with a new computer if and when that happens much easier. If you have a YubiKey like me, you can set the FIDO2 PIN using the [YubiKey Manager](https://www.yubico.com/support/download/yubikey-manager/) software.\n\n## Adding the key to GitLab\nNow that I had the complex parts covered, all that was left was to add the key to GitLab. I went to my [SSH settings](https://gitlab.com/-/profile/keys) on GitLab.com and (bravely) deleted my old SSH key and added the `.pub` public part of my key to my profile.\n\nAnd it was that simple! Now every time I go to interact with GitLab.com, I'm prompted to confirm my presence by touching the YubiKey device attached to my computer:\n\n```bash\ngit clone git@gitlab.com:brendan/website.git\nCloning into 'website'...\nConfirm user presence for key ECDSA-SK SHA256:OZSZGwbnnbc...\n\ngit add .\ngit commit -m \"A new commit\"\ngit push\nConfirm user presence for key ECDSA-SK SHA256:OZSZGwbnnbc...\n```\n\nThat small but essential change gives me peace of mind that even if someone could somehow get my private SSH key, I would still be protected by having physical access restricted to my YubiKey.\n\n",[789,830,9],{"slug":2802,"featured":6,"template":700},"how-to-protect-gitlab-connected-ssh-key-with-yubikey","content:en-us:blog:how-to-protect-gitlab-connected-ssh-key-with-yubikey.yml","How To Protect Gitlab Connected Ssh Key With Yubikey","en-us/blog/how-to-protect-gitlab-connected-ssh-key-with-yubikey.yml","en-us/blog/how-to-protect-gitlab-connected-ssh-key-with-yubikey",{"_path":2808,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2809,"content":2814,"config":2820,"_id":2822,"_type":13,"title":2823,"_source":15,"_file":2824,"_stem":2825,"_extension":18},"/en-us/blog/how-to-read-open-source-finding-middleman-callbacks",{"title":2810,"description":2811,"ogTitle":2810,"ogDescription":2811,"noIndex":6,"ogImage":2591,"ogUrl":2812,"ogSiteName":685,"ogType":686,"canonicalUrls":2812,"schema":2813},"How to Read Open Source: Finding Middleman Callbacks","Open source empowers you to learn beyond existing documentation. Getting started can be confusing. This is a demonstration finding unlisted Middleman callbacks.","https://about.gitlab.com/blog/how-to-read-open-source-finding-middleman-callbacks","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to Read Open Source: Finding Middleman Callbacks\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tyler Williams\"}],\n        \"datePublished\": \"2021-01-20\",\n      }",{"title":2810,"description":2811,"authors":2815,"heroImage":2591,"date":2817,"body":2818,"category":2600,"tags":2819},[2816],"Tyler Williams","2021-01-20","        \n\n{::options parse_block_html=\"true\" /}\n\n\n\n## Why read open source? \n\nWhen folks write about open source, I think there is a strong emphasis on contributing to open source projects, which makes sense. Many software professionals are excited to give back to the community. \n\nBut beyond adding to your favorite project, open source philosophies have a number of other benefits in our daily lives. In particular, I love open source because it allows me to learn more about my tools when the documentation is out of date, incomplete, or leaves me with additional questions from my own curiosity. \n\nThis happened to me recently when I was working on [https://about.gitlab.com](https://about.gitlab.com), a static site built with [Middleman](https://middlemanapp.com/). I needed to find a more comprehensive list of available [callbacks](https://middlemanapp.com/advanced/custom-extensions/#callbacks) in the Middleman lifecycle.\n\nI hope this blog post is helpful if you're looking for existing Middleman callbacks, or if you're getting started reading through the source code of your favorite open source tools. \n\n## The task at hand\n\nIf you're getting started reading open source, I find it helps to have a specific task. Any unfamiliar codebase can be challenging to navigate. Having a goal in mind narrows your focus. Here was my task for Middleman:\n\nI recently created a merge request to [add Webpack devServer to the local development environment](https://gitlab.com/gitlab-com/www-gitlab-com/-/merge_requests/71845). I had to modify some existing behavior of our Middleman preview server and wanted to use one of the lifecycle callbacks to modify the preview server's log output. \n\nHowever, the Middleman documentation does not currently list all available callbacks, nor where they happen in the lifecycle. The [extension docs](https://middlemanapp.com/advanced/custom-extensions/) say: \n\n> Middleman extensions are Ruby classes which can hook into various points of the Middleman system, add new features and manipulate content. This guide explains some of what's available, but you should read the Middleman source and the source of plugins like middleman-blog to discover all the hooks and extension points.\n\nI took them up on their advice and read through the [Middleman source code](https://github.com/middleman/middleman) to find the available callbacks. Here's what I found, and how I found them.\n\n## Callbacks available in Middleman Core\n\n1. `initialized`: called before config is parsed, and before extensions are registered\n1. `configure`: called to run any `configure` blocks (once for current environment, again for the current mode)\n1. `before_extensions`: called before the `ExtensionManager` is instantiated\n1. `before_instance_block`: called before any blocks are passed to the configuration context\n1. `before_sitemap`: called before the `SiteMap::Store` is instantiated, which initializes the sitemap\n1. `before_configuration`: called before configuration is parsed, mostly used for extensions\n1. `after_configuration`: called after extensions have worked\n1. `after_configuration_eval`: called after the configuration is parsed, before the pre-extension callback\n1. `ready`: called when everything is stable\n1. `before_build`: called before the site build process runs\n1. `after_build`: called after the builder is complete\n1. `before_shutdown`: called in the `shutdown!` method, which lets users know the application is shutting down\n1. `before`: called before Rack requests\n1. `before_server`: called before the `PreviewServer` is created\n1. `reload`: called before the new application is initialized on a reload event\n\n## How to find Middleman Callbacks\n\n1. [Clone](https://docs.gitlab.com/ee/gitlab-basics/start-using-git.html) the [Middleman repository](https://github.com/middleman/middleman) to your local machine. \n1. Open the Middleman directory in a text editor, IDE, or any tool that allows you to easily search through a folder's files for specific strings. \n1. Start with the [existing documentation](https://middlemanapp.com/advanced/custom-extensions/#callbacks) if it exists. Middleman lists the names of a few callbacks. You can search the directory for the string `after_configuration`.\n1. In this instance, you should be able to find that string used like so: `execute_callbacks(:after_configuration)`.\n1. You may also find it listed with other similar symbols in `middleman-core/lib/middleman-core/application.rb`. \n1. Read through the related blocks of code around these search results, you'll get some additional context for how they work, and you may find additional search terms that will be helpful.\n1. In the case of Middleman callbacks, you can continue to search for combinations of the `execute_callbacks` method with any callback listed in `middleman-core/lib/middleman-core/application.rb` to find where and when specific callbacks are used.\n\n## Contribute!\n\nWith this in-depth knowledge of your tool, you can be more productive at your day-to-day work, and you can contribute back to open source more easily. \n\nIn my case, I was able to identify the best callback for my usecase, and I now plan to check in with the Middleman team and ask if they would accept a contribution to their documentation with this information so it's easier for other folks to find in the future.\n",[743,9,956],{"slug":2821,"featured":6,"template":700},"how-to-read-open-source-finding-middleman-callbacks","content:en-us:blog:how-to-read-open-source-finding-middleman-callbacks.yml","How To Read Open Source Finding Middleman Callbacks","en-us/blog/how-to-read-open-source-finding-middleman-callbacks.yml","en-us/blog/how-to-read-open-source-finding-middleman-callbacks",{"_path":2827,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2828,"content":2834,"config":2839,"_id":2841,"_type":13,"title":2842,"_source":15,"_file":2843,"_stem":2844,"_extension":18},"/en-us/blog/how-to-scan-a-full-commit-history-to-detect-sensitive-secrets",{"title":2829,"description":2830,"ogTitle":2829,"ogDescription":2830,"noIndex":6,"ogImage":2831,"ogUrl":2832,"ogSiteName":685,"ogType":686,"canonicalUrls":2832,"schema":2833},"How to scan a full commit history to detect sensitive secrets","Use GitLab Secret Detection to scan a repository's commit history, including branches. View results within the GitLab UI with just a few lines of code added to a pipeline file.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097948/Blog/Hero%20Images/Blog/Hero%20Images/REFERENCE%20-%20display%20preview%20for%20blog%20images%20%281%29_2XDPsbkjQ3o6tcdom6IGxI_1750097948673.png","https://about.gitlab.com/blog/how-to-scan-a-full-commit-history-to-detect-sensitive-secrets","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to scan a full commit history to detect sensitive secrets\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Noah Ing\"},{\"@type\":\"Person\",\"name\":\"Jerez Solis\"}],\n        \"datePublished\": \"2025-02-06\",\n      }",{"title":2829,"description":2830,"authors":2835,"heroImage":2831,"date":2836,"body":2837,"category":830,"tags":2838},[847,1894],"2025-02-06","Secrets left exposed in outdated repositories pose significant risk for data\nbreaches. For example, a still-active secret key can be exposed, leaving it\nvulnerable to exploitation. Secrets include access keys, API tokens, private\nkeys, and other sensitive values. \n\n\nIn this article, you'll learn how to use GitLab Secret Detection to scan a\nrepository’s full commit history, including all branches, to detect\nsensitive secrets. In addition, you will discover how to view the results\ndirectly within the GitLab UI without the need for any integration. All it\ntakes is just a couple of lines of code in your `.gitlab-ci.yml` pipeline\nfile. \n\n\n## Scan every corner of your repository\n\n\nWe will use the sample repository shown in the screenshot below as an\nexample. To keep things simple, there is only a `README.md` file present in\nthe default branch of this repository. \n\n\n![Sample repository to\nscan](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097956/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097955851.png)\n\n\nAt first glance, it may seem like the repository is empty and that there are\nprobably no sensitive secrets in this repository. But what we are looking at\nis only the state of the default branch, which is the main branch in this\nexample. There could be feature branches in this repository created weeks,\nmonths, or years ago with sensitive secrets. It is also possible that a file\nwith a secret was accidentally pushed to the repo and then deleted right\nafter. However, it likely was not deleted correctly and is still in the\ncommit history.\n\n\nWe are going to enable GitLab Secret Detection scanner and set the\n`SECRET_DETECTION_HISTORIC_SCAN` variable to **true** so that the content of\nall branches in the repository is scanned.\n\n\n![Enable GitLab Secret Detection variable to\ntrue](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097956/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097955853.png)\n\n\n```\n\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\nsecret_detection:\n  variables:\n    SECRET_DETECTION_HISTORIC_SCAN: \"true\"\n```\n\n\nBy setting the `SECRET_DETECTION_HISTORIC_SCAN` variable to **true**, GitLab\nSecret Detection looks into every branch and commit of your repository. It\nensures that no sensitive information — whether from a feature branch or an\nold commit — is left unchecked.\n\n\n## Results of the scan\n\n\nTwo sensitive secrets were identified in the repository. One is a password\nin a `.env` file that was deleted from the repository, but the commit\ncontaining it was not removed from the git history. The other is an AWS\nAccess Token found in a feature branch. These exposed secrets could\ncompromise the organization’s security. \n\n\n![AWS Access Token\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097956/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097955855.png)\n\n\nYou can click on the AWS Access Token result to see more details, including\nthe file location. You can also create a GitLab issue to triage the\nvulnerability with one click. If you’re using the Jira integration, you can\ncreate a Jira ticket directly from the vulnerability page as well.\n\n\n## Why scanning for secrets matters\n\n\nAnyone with access to the repository can misuse the secret to gain\nunauthorized access to private resources and sensitive data. \n\n\nIn addition to scanning a repository’s full commit history across all\nbranches, GitLab Secret Detection also helps you take a multilayered\napproach to detecting secrets:\n\n\n* [Secret push\nprotection](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/index.html)\n- scans commits for secrets during a push and blocks it if secrets are\ndetected, unless skipped, reducing the risk of leaks.  \n\n* [Pipeline secret\ndetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/index.html)\n- scans files after they’ve been committed and pushed to a GitLab\nrepository.\n\n* [Client-side secret\ndetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/client/index.html)\n- scans comments and descriptions in issues and merge requests for secrets\nbefore they're saved to GitLab.  * [Automatic response to leaked\nsecrets](https://docs.gitlab.com/ee/user/application_security/secret_detection/automatic_response.html)\n- automatically revokes certain types of leaked secrets and notifies the\npartner that issued the secret. \n\n\nYou can adjust pipeline secret detection to suit your needs by modifying,\nextending, or replacing the default ruleset. For instance, you can define\n[custom\nrules](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/index.html#customize-analyzer-rulesets)\nusing regex patterns to detect sensitive data like credit card numbers,\nphone numbers, or other information specific to your organization.\n\n\n## Try GitLab Secret Detection\n\n\n1.\n[Enable](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/#enable-the-analyzer)\nSecret Detection in your GitLab pipeline.  \n\n2. Set `SECRET_DETECTION_HISTORIC_SCAN: true`.  \n\n3. Push and trigger a pipeline to scan all branches and commits.\n\n\nGitLab makes securing your code simple and comprehensive. Don’t let an old\nbranch or commit compromise your security — give historical scans a try\ntoday!\n\n\n> #### [Sign up for a free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/) to get started with security\nscanners like Secret Detection.\n",[1016,9,495,696],{"slug":2840,"featured":6,"template":700},"how-to-scan-a-full-commit-history-to-detect-sensitive-secrets","content:en-us:blog:how-to-scan-a-full-commit-history-to-detect-sensitive-secrets.yml","How To Scan A Full Commit History To Detect Sensitive Secrets","en-us/blog/how-to-scan-a-full-commit-history-to-detect-sensitive-secrets.yml","en-us/blog/how-to-scan-a-full-commit-history-to-detect-sensitive-secrets",{"_path":2846,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2847,"content":2853,"config":2858,"_id":2860,"_type":13,"title":2861,"_source":15,"_file":2862,"_stem":2863,"_extension":18},"/en-us/blog/how-to-secure-cloud-run-deployment-with-auto-devops",{"title":2848,"description":2849,"ogTitle":2848,"ogDescription":2849,"noIndex":6,"ogImage":2850,"ogUrl":2851,"ogSiteName":685,"ogType":686,"canonicalUrls":2851,"schema":2852},"How to secure Google Cloud Run deployment with GitLab Auto DevOps","This tutorial will help teams speed development, improve security, and harness the power of serverless technology.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682898/Blog/Hero%20Images/cloud-security.png","https://about.gitlab.com/blog/how-to-secure-cloud-run-deployment-with-auto-devops","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to secure Google Cloud Run deployment with GitLab Auto DevOps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Regnard Raquedan\"}],\n        \"datePublished\": \"2023-08-21\",\n      }",{"title":2848,"description":2849,"authors":2854,"heroImage":2850,"date":2855,"body":2856,"category":785,"tags":2857},[1874],"2023-08-21","\nTeams looking for efficiency often look to GitLab and serverless platforms to minimize management overhead and speed deployment times. GitLab's tight integration with [Google Cloud Run](https://cloud.google.com/run) means that teams can take advantage of the industry-leading DevSecOps platform to deliver container-based applications securely and efficiently.\n\nThis tutorial will show you how to deploy applications to Cloud Run using GitLab [Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/), a feature that lets developers quickly use CI/CD pipelines via pre-built templates. This approach can accelerate testing and deployment because stages and jobs are already pre-configured.\n\n## Prerequisites\nBefore you begin, make sure you have the following:\n- a Google Cloud project with Cloud Run and Cloud Build APIs enabled\n- a Google Cloud service account with Cloud Run Admin, Cloud Build Service Agent, Service Account User, and Project Viewer permissions\n- a GitLab project containing your application code\n\n### Demo walkthrough\n\n\u003Ciframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/hIFagDyo3f8\" title=\"YouTube video player\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" allowfullscreen>\u003C/iframe>\n\n\n**Step 1:** Configure Google Cloud credentials\n\nTo start, use the Google Cloud service account with the necessary permissions. Once you have the service account, export its key to a JSON file and encode it using base64.\n\n**Step 2:** Add Auto DevOps to your GitLab project\n\nNavigate to your GitLab project and create a new file at the root called \"gitlab-ci.yml.\" Add the following lines of code to include the Auto DevOps template, which automatically configures your pipeline based on project settings and configuration:\n\n```\ninclude:\n  - template: Auto-DevOps.gitlab-ci.yml\n```\n\nCommit the changes to your project.\n\n**Step 3:** Configure environment variables\n\nAdd the following environment variables to your GitLab project:\n\n* `BASE64_GOOGLE_CLOUD_CREDENTIALS`: The base64-encoded JSON file containing your service account key. Make sure to mask this variable.\n* `PROJECT_ID`: The Google Cloud project ID.\n* `SERVICE_ID`: The service ID that will be used for Cloud Run. For this tutorial, we'll use \"nodejs\" as our service ID.\n\n**Step 4:** Configure the CI/CD pipeline\n\nModify the \"gitlab-ci.yml\" file to add Google Cloud SDK, gcloud commands, Docker, and the necessary configurations for deploying your application to Cloud Run. \n\n```\nimage: google/cloud-sdk:latest\n```\n\nAdditionally, use Google Cloud Build to generate the container image required for deployment. Commit the changes to your project.\n\n```\ndeploy:\n  stage: deploy\n  script:\n    - export GOOGLE_CLOUD_CREDENTIALS=$(echo $BASE64_GOOGLE_CLOUD_CREDENTIALS | base64 -d)\n    - echo $GOOGLE_CLOUD_CREDENTIALS > service-account-key.json \n    - gcloud auth activate-service-account --key-file service-account-key.json \n    - gcloud config set project $PROJECT_ID \n    - gcloud auth configure-docker\n    - gcloud builds submit --tag gcr.io/$PROJECT_ID/$SERVICE_ID\n    - gcloud run deploy $SERVICE_ID --image gcr.io/$PROJECT_ID/$SERVICE_ID --region=us-central1 --platform managed --allow-unauthenticated \n```\n\n**Step 5:** Finalize the DAST stage\n\nOnce your application has been deployed to Cloud Run, complete the dynamic application security testing ([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)) stage in the CI/CD pipeline to ensure your application is more secure. Add the Cloud Run URL to your \"gitlab-ci.yml\" file and enable full_scan and browser_scan options. Commit the changes to your project.\n\n```\nvariables:\n  DAST_WEBSITE: \u003Cproject URL>\n  DAST_FULL_SCAN_ENABLED: \"true\"\n  DAST_BROWSER_SCAN: \"true\" \n```\n\nIn this tutorial, we successfully deployed a Cloud Run application using GitLab's Auto DevOps. By following these steps, you can enjoy faster development and improved security, and harness the power of serverless technology.\n",[9,1898,830],{"slug":2859,"featured":6,"template":700},"how-to-secure-cloud-run-deployment-with-auto-devops","content:en-us:blog:how-to-secure-cloud-run-deployment-with-auto-devops.yml","How To Secure Cloud Run Deployment With Auto Devops","en-us/blog/how-to-secure-cloud-run-deployment-with-auto-devops.yml","en-us/blog/how-to-secure-cloud-run-deployment-with-auto-devops",{"_path":2865,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2866,"content":2872,"config":2878,"_id":2880,"_type":13,"title":2881,"_source":15,"_file":2882,"_stem":2883,"_extension":18},"/en-us/blog/how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes",{"title":2867,"description":2868,"ogTitle":2867,"ogDescription":2868,"noIndex":6,"ogImage":2869,"ogUrl":2870,"ogSiteName":685,"ogType":686,"canonicalUrls":2870,"schema":2871},"How to stream logs through the GitLab Dashboard for Kubernetes","In GitLab 17.2, users can now view Kubernetes pod and container logs directly via the GitLab UI. This tutorial shows how to use this new feature to simplify monitoring Kubernetes infrastructure.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662245/Blog/Hero%20Images/blog-image-template-1800x945__16_.png","https://about.gitlab.com/blog/how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to stream logs through the GitLab Dashboard for Kubernetes\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Daniel Helfand\"}],\n        \"datePublished\": \"2024-08-19\",\n      }",{"title":2867,"description":2868,"authors":2873,"heroImage":2869,"date":2875,"body":2876,"category":785,"tags":2877},[2874],"Daniel Helfand","2024-08-19","Developers are context-switching more frequently, needing to understand and\nuse multiple tools to accomplish complex tasks. These tools all have\ndifferent user experiences and often do not present all the information\nneeded to successfully develop, troubleshoot, and ship critical features. It\nis challenging enough to release and monitor software changes without also\nneeding to understand so many tools.\n\n\nWith the addition of [pod log streaming through the GitLab Dashboard for\nKubernetes in\nv17.2](https://about.gitlab.com/releases/2024/07/18/gitlab-17-2-released/#log-streaming-for-kubernetes-pods-and-containers),\ndevelopers can go straight from a merge request review to watching a\ndeployment rolled out to Kubernetes. This new feature will:\n\n- allow developers to avoid switching tooling\n\n- ease the process of troubleshooting and monitoring deployments and\npost-deployment application health\n\n- strengthen [GitOps\nworkflows](https://docs.gitlab.com/ee/user/clusters/agent/gitops.html) to\neasily manage application and infrastructure changes\n\n\nThe new feature allows GitLab users to view the logs of pods and containers\ndirectly via the GitLab UI. In previous versions of GitLab, users could\nconfigure a GitLab project to view pods deployed to certain namespaces on an\nassociated cluster. This new feature allows users to further monitor\nworkloads running on Kubernetes without needing to switch to another tool.\n\n\nIn the sections below, you will learn how to use this new feature by adding\na Kubernetes cluster to a GitLab project, deploying a sample workload to a\ncluster, and viewing the logs of this workload running on a cluster. \n\n\n> Need to know the basics of Kubernetes? [Read this quick introductory\nblog](https://about.gitlab.com/blog/kubernetes-the-container-orchestration-solution/).\n\n\n## Configure a GitLab project to view Kubernetes resources\n\n\nBefore proceeding with this section, the following prerequisites are\nrequired:\n\n* a remote Kubernetes cluster (i.e., not running locally on your machine)\n\n* access to a GitLab v17.2 account\n\n* [this\nrepository](https://gitlab.com/gitlab-da/tutorials/cloud-native/gitlab-k8s-log-streaming-example)\nforked to a GitLab group to which you have access\n\n* Helm CLI\n\n* kubectl CLI\n\n\nOnce you have satisfied the prerequisites involved, add an agent\nconfiguration file to the GitLab project you forked. The configuration file\nallows users to control permissions around how GitLab users may interact\nwith the associated Kubernetes cluster.\n\n\nYou can use the configuration file included in this GitLab project by\nchanging the following file: `.gitlab/agents/k8s-agent/config.yaml`. Replace\nthe `\u003CGitLab group>` in the id property shown below with the group where you\nhave forked the example project. This config file will allow [GitLab to\naccess your cluster via an\nagent](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html) that\ncan be installed on your cluster.\n\n\n```yaml\n\nuser_access:\n  access_as:\n    agent: {}\n  projects:\n    - id: \u003CGitLab group>/gitlab-k8s-log-streaming-example\n```\n\n\nOnce the above file is edited, you can commit and push these changes to the\nmain branch of the project. \n\n\n## Add GitLab Kubernetes agent to cluster\n\n\nWith the agent configuration file added, now add the cluster to GitLab by\ninstalling an agent on your cluster. In the GitLab UI, go to your project\nand, on the left side of the screen, select **Operate > Kubernetes\nclusters**. Once on this page, select the **Connect a cluster** button on\nthe right side of the screen. From the dropdown menu, you can then select\nthe agent, which should be `k8s-agent`. Click **Register** to get\ninstructions for how to install the agent on your cluster.\n\n\nThe instructions presented to you after registering the agent will be to run\na helm command that will install the GitLab agent on your cluster. Before\nrunning the command locally, you will want to ensure your Kubernetes context\nis targeting the cluster you want to work with. Once you have verified you\nare using the correct kubeconfig locally, you can run the helm command to\ninstall the agent on your cluster.\n\n\nOnce both pods are running, GitLab should be able to connect to the agent.\nRun the following command to wait for the pods to start up:\n\n\n```shell\n\nkubectl get pods -n gitlab-agent-k8s-agent -w\n\n```\n\n\n## Deploy sample application to your cluster\n\n\nBefore you can view logs of a workload through GitLab, you first need to\nhave something running on your cluster. To do this, you can run the\nfollowing kubectl command locally. \n\n\n```shell\n\nkubectl apply -f\nhttps://gitlab.com/gitlab-da/tutorials/cloud-native/gitlab-k8s-log-streaming-example/-/raw/main/k8s-manifests/k8s.yaml\n\n```\n\n\nAfter the command runs successfully, you are now ready to complete the final\nstep to set up a Kubernetes dashboard via GitLab.\n\n\n## View pod logs through the GitLab UI\n\n\nTo add the Kubernetes dashboard via the GitLab UI, go to your project and,\non the left side of the screen, select **Operate > Environments**. On the\ntop right side of the screen, select the **Create an environment**.\n\n\nNext, you can give your environment a name, select the GitLab agent (i.e.\n`k8s-agent`), and pick a namespace for the Kubernetes dashboard to focus on.\nSince the application is running in the\n`gitlab-k8s-log-streaming-example-dev` namespace, select this option from\nthe namespace dropdown. After naming the environment and selecting the agent\nand namespace, click **Save**.\n\n\nAfter creating the environment, you should now see information about the\napplication’s pods displayed via the GitLab UI.\n\n\n![Kubernetes logs - image\n2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676402/Blog/Content%20Images/Screenshot_2024-08-20_at_12.15.08_PM.png)\n\n\nGo to the right side of the screen and click **View Logs** to see logs for\none of the pods associated with the application. \n\n\n![Kubernetes dashboard - image\n1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676402/Blog/Content%20Images/Screenshot_2024-08-20_at_12.16.56_PM.png)\n\n\n## Try it out and share feedback\n\n\nThe introduction of pod log streaming in GitLab v17.2 will help GitLab users\nget one step closer to managing complex deployments to Kubernetes, as well\nas monitoring and troubleshooting issues post deployment via a common user\nexperience. We are excited to hear more about users’ experiences with this\nnew enhancement and how it helps improve DevOps workflows around Kubernetes.\nTo share your experience with us, you can open an issue to the [project\nassociated with this\ntutorial](https://gitlab.com/gitlab-da/tutorials/cloud-native/gitlab-k8s-log-streaming-example).\nOr, [comment directly in the Kubernetes log streaming feedback\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/478379) to report\ninformation to the GitLab engineering team.\n\n\nMore information on getting started with the GitLab Dashboard for Kubernetes\ncan be found in the documentation\n[here](https://docs.gitlab.com/ee/ci/environments/kubernetes_dashboard.html).\n\n\n> To explore the GitLab Dashboard for Kubernetes as well as other more\nadvanced features of GitLab, sign up for [our free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).\n",[696,549,1241,9],{"slug":2879,"featured":90,"template":700},"how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes","content:en-us:blog:how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes.yml","How To Stream Logs Through The Gitlab Dashboard For Kubernetes","en-us/blog/how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes.yml","en-us/blog/how-to-stream-logs-through-the-gitlab-dashboard-for-kubernetes",{"_path":2885,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2886,"content":2892,"config":2897,"_id":2899,"_type":13,"title":2900,"_source":15,"_file":2901,"_stem":2902,"_extension":18},"/en-us/blog/how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags",{"title":2887,"description":2888,"ogTitle":2887,"ogDescription":2888,"noIndex":6,"ogImage":2889,"ogUrl":2890,"ogSiteName":685,"ogType":686,"canonicalUrls":2890,"schema":2891},"How to translate Bamboo agent capabilities to GitLab Runner tags  ","This tutorial demonstrates how to use tags to organize GitLab Runners when building complex CI/CD pipelines.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663019/Blog/Hero%20Images/AdobeStock_519147119.jpg","https://about.gitlab.com/blog/how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to translate Bamboo agent capabilities to GitLab Runner tags  \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2024-02-22\",\n      }",{"title":2887,"description":2888,"authors":2893,"heroImage":2889,"date":2894,"body":2895,"category":785,"tags":2896},[1604],"2024-02-22","CI pipelines often start simple – a single job building a binary and pushing\nit to an artifact repository or to some production environment.\nEver-changing software requirements introduce more complexities, such as\nadding more jobs to perform certain checks and reviewing the output before\nthe final build job is executed.  \n\n\nThese complexities increase exponentially when builds are expected to target\nvarying systems with different system architectures or resource needs. This\nis evident in projects like operating systems, mobile apps, or software\ndistributions that support multiple deployment platforms. To account for the\nvarying needs of builds in these types of environments, having multiple\nrunners that match needed requirements is key, and that's where [GitLab\nRunner](https://docs.gitlab.com/runner/) tags come in. If you are coming\nfrom Atlassian's Bamboo, they are called \"agent capabilities.\"\n\n\nRunner tags allow organizing runners by a tag that signifies a specific use\ncase they support; these tags are then used to make sure CI jobs run on a\nrunner that meets their requirements. A job can require GPU resources that\nare only available on a handful of runners; tagging the job to the tags of\nthe runner allows it to be scheduled on the runner with GPUs.\n\n\nAgent capabilities on Bamboo are used to achieve the same functionality by\nspecifying binaries or custom identifiers that must be matched or available\nfor a job to run on a Bamboo agent. In this blog post, we will be looking at\nhow to translate Bamboo agent capabilities to GitLab Runner tags. \n\n\nBamboo has varying agent capabilities:\n\n- Executable capability specifies executables that are available on an\nagent.\n\n- JDK capability specifies that the Java Development Kit is installed and\navailable for builds.\n\n- Version Control capability lets Bamboo know the version control systems\nset up on an agent and where the client application is located.\n\n- Docker capability is used to define the agents where Docker is installed\nfor Docker tasks\n\n- Custom capability uses key/value identifiers to identify a unique\nfunctionality an agent provides.\n\n\nGitLab makes the process easier by using tags to identify Runners, some of\nwhich can be assigned multiple tags to denote the varying functionalities\nthey can provide to jobs. Let's look at how you can use Runner tags in\nGitLab.\n\n\n## Adding tags to GitLab Runner\n\n\nWhen [registering a\nrunner](https://docs.gitlab.com/runner/register/index.html) after\ninstallation, one of the steps requires providing a list of comma-separated\ntags that can be used. If none are provided at this stage, you can always\nedit the `/etc/gitlab-runner/config.toml` file and add any missing tags.\n\n\nYou can also manage the tags of a runner in GitLab by accessing the runner's\nedit page and updating the `Tags` field. You have the option for the runner\nto be exclusive to jobs that are tagged appropriately, or when there are no\ntagged jobs to run, it should run untagged jobs, too. Checking `Run untagged\njobs` enables this behavior.\n\n\n## Using tags in .gitlab-ci.yaml file\n\n\nTo run a job on a specific runner, add the relevant tags to the job's\nconfiguration, as shown below:\n\n\n```yaml\n\nbuild_ios:\n  image: macos-13-xcode-14\n  stage: build\n  script:\n    - bundle check --path vendor/bundle || bundle install --path vendor/bundle --jobs $(nproc)\n    - bundle exec fastlane build\n  tags: \n    - saas-macos-medium-m1\n```\n\nIn the example above, the job builds an iOS application only on runners\noperating on a macOS device with an M1 chip and tagged\n`saas-macos-medium-m1`.\n\n\n## Using multiple tags\n\n\nA job can specify multiple tags to target a diverse range of runners,\nespecially in organizations that run several fleets of runners as part of\ntheir software development lifecycle. A job will only run if a runner is\nfound that has all the tags the job has been tagged with. For example, if a\njob has `[linux, android, fastlane]` tags, a runner with `[ android,\nfastlane]` or `[ linux, android]` will not execute the job because the full\nset of tags does not match the runner.\n\n\n## Dynamic jobs with tags and variables\n\n\nYou can use variables to determine the values of tags and thus dynamically\ninfluence which runners pick up the jobs. For example:\n\n\n```\n\nvariables:\n  KUBERNETES_RUNNER: kubernetes\n\n  job:\n    tags:\n      - docker\n      - $KUBERNETES_RUNNER\n    script:\n      - echo \"Hello runner selector feature\"\n\n``` \n\n\nIn this example, only runners tagged with `kubernetes` will execute the job.\nYou can take this further in more complex pipelines with [`parallel:\nmatrix`](https://docs.gitlab.com/ee/ci/yaml/index.html#parallelmatrix). Here\nis an example:\n\n\n```\n\ndeploystacks:\n  stage: deploy\n  parallel:\n    matrix:\n      - PROVIDER: aws\n        STACK: [monitoring, app1]\n      - PROVIDER: gcp\n        STACK: [data]\n  tags:\n    - ${PROVIDER}-${STACK}\n  environment: $PROVIDER/$STACK\n\n```\n\n\nThis example ends up with three parallel jobs with three different tags for\neach: `aws-monitoring`, `aws-app1` and `gcp-data`, thus targeting possibly\nthree different runners.\n\n\nUsing tags in your GitLab CI configuration gives you the flexibility to\ndetermine where and how your applications are built, to use resources more\nefficiently as scarce resources can be limited to certain runners, and to\ndetermine how jobs are allocated to those runners.\n\n\n> Learn more about [how to make the move from Atlassian to\nGitLab](https://about.gitlab.com/move-to-gitlab-from-atlassian/).\n",[108,787,9],{"slug":2898,"featured":90,"template":700},"how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags","content:en-us:blog:how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags.yml","How To Translate Bamboo Agent Capabilities To Gitlab Runner Tags","en-us/blog/how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags.yml","en-us/blog/how-to-translate-bamboo-agent-capabilities-to-gitlab-runner-tags",{"_path":2904,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2905,"content":2911,"config":2916,"_id":2918,"_type":13,"title":2919,"_source":15,"_file":2920,"_stem":2921,"_extension":18},"/en-us/blog/how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops",{"title":2906,"description":2907,"ogTitle":2906,"ogDescription":2907,"noIndex":6,"ogImage":2908,"ogUrl":2909,"ogSiteName":685,"ogType":686,"canonicalUrls":2909,"schema":2910},"How to use GitLab's Custom Compliance Frameworks in your DevSecOps environment","Explore how new frameworks, along with more than 50 out-of-the-box controls, transform regulatory requirements from burdensome checkboxes to integrated, automated workflow components.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097104/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%284%29_3LZkiDjHLjhqEkvOvBsVKp_1750097104092.png","https://about.gitlab.com/blog/how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use GitLab's Custom Compliance Frameworks in your DevSecOps environment\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-04-30\",\n      }",{"title":2906,"description":2907,"authors":2912,"heroImage":2908,"date":2913,"body":2914,"category":830,"tags":2915},[1444],"2025-04-30","Compliance isn't just a checkbox — it's a critical business function that affects everything from operational risk to customer trust. For development teams, balancing compliance requirements with velocity can be particularly challenging. GitLab's [Custom Compliance Frameworks](https://about.gitlab.com/blog/introducing-custom-compliance-frameworks-in-gitlab/) offer a powerful way to integrate compliance verification directly into your development workflow. In this article you'll learn what they are and how to use them for maximum efficiecy.\n\n## What are GitLab Custom Compliance Frameworks?\n\nGitLab Custom Compliance Frameworks allow organizations to define, implement, and enforce compliance standards directly within their GitLab instance. This feature extends GitLab's built-in compliance capabilities by enabling teams to create customized frameworks that align with specific regulatory requirements, internal policies, or industry standards.\n\nCustom Compliance Frameworks have the following benefits:\n* Reduce manual tracking  \n* Accelerate audit readiness  \n* Enforce compliance controls natively\n\n![Compliance center screenshot with frameworks listed](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097114254.png)\n\nWith this release, more than 50 out-of-the-box (OOTB) controls are provided (with more coming soon) that can be tailored to your organization's unique compliance needs, including HIPAA in healthcare, GDPR for data privacy, SOC2 for service organizations, or industry-specific regulations. Some examples of OOTB controls include:\n\n* Separation of duties (e.g., at least two approvers and author approved merge request)  \n* Security scanners running (e.g., [SAST](https://docs.gitlab.com/user/application_security/sast/) running and [Dependency Scanning](https://docs.gitlab.com/user/application_security/dependency_scanning/) running)  \n* Authentication/authorization (e.g., project visibility not public and AuthSSO required)  \n* Application configuration (e.g., status checks required and Terraform required)\n\nAdditionally, you can configure external environmental controls using the GitLab API to check the status and details of an external environment.\n\n## Creating a Custom Compliance Framework from scratch\n\nNow that we understand the value, let's explore how to implement Custom Compliance Frameworks in your GitLab environment. We will use this demo application and you can follow along in this video. \n\n**Note:** A GitLab Ultimate subscription is required.\n\n\u003C!-- TODO: EMBED_YT_VIDEO -->\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/bSwwv5XeMdQ?si=unDwCltF4vTHT4mB\" title=\"Adhering to compliance requirements with built-in compliance controls\n\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n**Step 1: Define your compliance requirements**\n\nBefore building your custom framework, you need to clearly define your compliance requirements:\n\n1. **Identify applicable regulations:** Determine which regulations and standards apply to your organization (e.g., GDPR, PCI DSS, and HIPAA). \n2. **Map requirements to controls:** Break down each regulation into specific, actionable controls.  \n3. **Prioritize requirements:** Focus on high-risk areas and requirements with the greatest impact.\n\n**Step 2: Create your Custom Compliance Framework**\n\nTo create a custom compliance framework in GitLab:\n\n1. Navigate to your GitLab group's **Secure > Compliance Center** section.  \n2. Press the **New framework** button.  \n3. Select **Create blank framework**.\n\n![Create a custom compliance framework screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097114255.png)\n\n4. Provide a name, description, and color for your framework.\n\n![New compliance framework screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097114257.png)\n\n5. Add a requirement to the framework:  \n   a. Scroll down to the **Requirements** tab.\n\n   b. Press the **New requirement** button.\n\n   c. Provide a name and description.  \n   d. Under the **Controls** section, select **Choose a GitLab control**.  \n   e. Select a control from the list (e.g., at least two approvals, SAST running).  \n   f. Press the **Create requirement** button.\n\n![Create new requirement button](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097114258.png)\n\n6. Press the **Create framework** button.\n\nThe framework will be created as specified and will now be available to add to projects. Additionally, compliance frameworks can be [imported](http://TODO) using a JSON with the appropriate schema.\n\n**Step 3: Apply the framework to projects**\n\nOnce your framework is created:\n1. From the Compliance Center, select the **Projects** tab.  \n2. Use the search bar to **Search** or **Filter** results.  \n3. Select the project(s) you wish to apply your framework to.  \n4. Press the **Choose one bulk action** button.  \n5. Select **Apply frameworks to selected projects**.  \n6. Press the **Select frameworks** button.  \n7. Select your framework(s) from the list.  \n8. Press the **Apply** button.\n\n![Compliance center screen with SOC 2 framework dropdown](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097114260.png)\n\nThe framework will now be applied to the project, making its requirements visible and trackable.\n\n**Step 4: Monitor and report on compliance**\n\nWith your framework in place, you can now:\n\n1. Use the **Compliance Center** to track compliance status across projects including details and suggested fixes for failed controls.\n2. Generate **compliance reports** for audits and stakeholder reviews.  \n3. Set up **compliance alerts** to notify stakeholders of potential compliance issues. \n4. Review **audit events** to overview action taken on compliance settings.\n\n![Compliance Center screen showing SOC2 test framework](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097114263.png)\n\n## Real-world example: Implement a SOC2 compliance framework\n\nSystem and Organization Controls 2, better known as SOC2, is a rigorous auditing standard developed by the American Institute of Certified Public Accountants that assesses a service organization's controls related to security, availability, processing integrity, confidentiality, and privacy. You can read my [Guide to fulfilling SOC 2 security requirements with GitLab](https://about.gitlab.com/blog/guide-to-fulfilling-soc-2-security-requirements-with-gitlab/) to learn more.\n\nNow, let's review a practical example of using a Custom Compliance Framework to verify SOC2 security compliance, which requires:\n\n* implementation of controls to protect against unauthorized access  \n* establishment of procedures for identifying and mitigating risks  \n* setting up systems for detecting and addressing security incidents\n\n**Disclaimer:** This is only an example showcasing some of the controls possible for adhering to SOC2. Be sure to consult with your security/compliance team before moving any implementation to production.\n\nA Custom Compliance Framework for SOC2 will look as follows using some GitLab OOTB controls:\n\n* **Name:** SOC2 Security Requirements  \n* **Description:** Adds the security requirements for SOC2 framework compliance  \n* **Requirements:**  \n  * **Implement controls to protect against unauthorized access**  \n    * Auth SSO enabled  \n    * CI/CD job token scope enabled  \n    * Require MFA at org level  \n  * **Establish procedures for identifying and mitigating risks**  \n    * At least two approvals  \n    * Author approved merge request  \n    * Committers approved merge request  \n    * Default branch protected  \n  * **Setting up systems for detecting and addressing security incidents**  \n    * Dependency Scanning running  \n    * SAST running  \n    * DAST running\n\nWhen applied to your project(s), this framework allows you to oversee if/and when they fall out of compliance and what can be done to bring them back into compliance. Note that you can create and apply multiple compliance frameworks to a project(s). For example, you can have one for SOC2 process integrity requirements.\n\n## Implement security policies to ensure compliance requirements are met\n\nAlthough not required, security policies can be applied to projects containing a Custom Compliance Framework. This allows you to assure that certain compliance criteria will be enforced via security policies. For example, you can force security scanners to run on projects that contain a Custom Compliance Framework requiring security scanning. \n\nGitLab provides various different security policies:\n\n* [Scan execution policy](https://docs.gitlab.com/user/application_security/policies/scan_execution_policies/): Enforces security scans, either as part of the pipeline or on a specified schedule.  \n* [Merge request approval policy](https://docs.gitlab.com/user/application_security/policies/merge_request_approval_policies/): Enforces project-level settings and approval rules based on scan results.  \n* [Pipeline execution policy](https://docs.gitlab.com/user/application_security/policies/pipeline_execution_policies/): Enforces CI/CD jobs as part of project pipelines. \n* [Vulnerability management policy](https://docs.gitlab.com/user/application_security/policies/vulnerability_management_policy/): Automatically resolves vulnerabilities that are no longer detected in the default branch.\n\nLet’s go ahead and force a SAST scanner to run in order to automatically adhere to any requirements that require SAST scanning. To create a security policy and apply it to a project with a particular framework:\n\n1. Navigate to a project that has a Custom Compliance Framework requiring **SAST scanning**. \n2. In the sidebar, select **Secure > Policies**.  \n3. Press the **New policy** button.  \n4. Under **Scan execution policy**, press the **Select policy** button. \n5. Fill in the **Name** and **Description**. \n6. Under **Actions**, select **SAST** as the scan to run.\n\n![Actions screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097114264.png)\n\n7. Under **Conditions**, select the pipeline to be triggered when a pipeline runs for all branches.\n\n![Conditions screen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097114/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097114265.png)\n\n8. Press the **Configure with a merge request** button.  \n9. An MR is now created in a separate project containing all the security policies scoped to this project.\n10. Press the **Merge** button.\n\nNow SAST will run for every branch, assuring you are compliant in that area. Be sure to review all the different types of security policies and see how they can suit your requirements.\n\n## 5 best practices to follow\n\nTo maximize the value of Custom Compliance Frameworks:\n\n1. **Start small:** Begin with one critical regulation or standard before expanding.  \n2. **Involve key stakeholders:** Include compliance, security, and development teams in framework creation.  \n3. **Automate where possible:** Use GitLab CI/CD to automate compliance checks.  \n4. **Document thoroughly:** Maintain clear documentation of how your framework maps to regulatory requirements.  \n5. **Review regularly:** Update your frameworks as regulations evolve or new requirements emerge.\n\n## Get started today\n\nGitLab Custom Compliance Frameworks represent a significant advancement in DevSecOps by bringing compliance directly into the development workflow. By implementing custom frameworks, organizations can reduce compliance overhead, improve risk management, and accelerate development cycles while maintaining robust compliance with regulatory requirements.\n\nThe ability to define and enforce Custom Compliance Frameworks gives teams the flexibility they need to address their specific regulatory landscape while providing the structure necessary to ensure consistent compliance practices across the organization.\n\nAs regulatory requirements continue to grow in complexity, tools like GitLab Custom Compliance Frameworks will become increasingly essential for organizations looking to balance compliance requirements with development velocity in a sustainable way.\n\n> To try Custom Compliance Frameworks today, sign up for your [free  trial of GitLab Ultimate](https://about.gitlab.com/free-trial/).\n\n## Learn more\n\nVisit these resources to learn more about Custom Compliance Frameworks and how they can benefit your organization:\n\n* [Custom Compliance Frameworks documentation](https://docs.gitlab.com/user/compliance/compliance_center/compliance_status_report/)  \n* [Custom Compliance Frameworks epic](https://gitlab.com/groups/gitlab-org/-/epics/13295)  \n* [Security Policies documentation](https://docs.gitlab.com/user/application_security/policies/)  \n* [GitLab Security and Compliance solutions](https://about.gitlab.com/solutions/security-compliance/)",[830,9,495,696,694],{"slug":2917,"featured":90,"template":700},"how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops","content:en-us:blog:how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops.yml","How To Use Gitlabs Custom Compliance Frameworks In Your Devsecops","en-us/blog/how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops.yml","en-us/blog/how-to-use-gitlabs-custom-compliance-frameworks-in-your-devsecops",{"_path":2923,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2924,"content":2930,"config":2935,"_id":2937,"_type":13,"title":2938,"_source":15,"_file":2939,"_stem":2940,"_extension":18},"/en-us/blog/how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery",{"title":2925,"description":2926,"ogTitle":2925,"ogDescription":2926,"noIndex":6,"ogImage":2927,"ogUrl":2928,"ogSiteName":685,"ogType":686,"canonicalUrls":2928,"schema":2929},"How to use OCI images as the source of truth for continuous delivery","Discover the benefits of using Open Container Initiative images as part of GitOps workflows and the many features GitLab offers to simplify deployments to Kubernetes.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097601/Blog/Hero%20Images/Blog/Hero%20Images/REFERENCE%20-%20Use%20this%20page%20as%20a%20reference%20for%20thumbnail%20sizes_76Tn5jFmEHY5LFj8RdDjNY_1750097600692.png","https://about.gitlab.com/blog/how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use OCI images as the source of truth for continuous delivery\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Daniel Helfand\"}],\n        \"datePublished\": \"2025-02-19\",\n      }",{"title":2925,"description":2926,"authors":2931,"heroImage":2927,"date":2932,"body":2933,"category":740,"tags":2934},[2874],"2025-02-19","Is [GitOps](https://about.gitlab.com/topics/gitops/) still GitOps if you are\nnot using a git repository as your deployment artifact? While git remains\ncentral to GitOps workflows, storing infrastructure definitions as Open\nContainer Initiative (OCI) artifacts in container registries has seen a rise\nin adoption as the source for GitOps deployments. In this article, we will\ndive deeper into the ideas behind this trend and how GitLab features support\nthis enhancement to GitOps workflows.\n\n\n## What is GitOps?\n\n\nThe [OpenGitOps](https://opengitops.dev/) project has defined [four\nprinciples](https://opengitops.dev/#principles) for the practice of GitOps:\n\n- A [system managed by\nGitOps](https://github.com/open-gitops/documents/blob/v1.0.0/GLOSSARY.md#software-system)\nmust have its [desired state expressed\ndeclaratively](https://github.com/open-gitops/documents/blob/v1.0.0/GLOSSARY.md#declarative-description).\n\n- Desired state is stored in a way that enforces immutability and\nversioning, and retains a complete version history.\n\n- Software agents automatically pull the desired state declarations from the\nsource.\n\n- Software agents\n[continuously](https://github.com/open-gitops/documents/blob/v1.0.0/GLOSSARY.md#continuous)\nobserve actual system state and [attempt to apply the desired\nstate](https://github.com/open-gitops/documents/blob/v1.0.0/GLOSSARY.md#reconciliation).\n\n\nAn example of GitOps is storing the Kubernetes manifests for a microservice\nin a GitLab project. Those Kubernetes resources are then continuously\nreconciled by a\n[controller](https://kubernetes.io/docs/concepts/architecture/controller/)\nrunning on the Kubernetes cluster where the microservice is deployed to.\nThis allows engineers to manage infrastructure using the same workflows as\nworking with regular code, such as opening merge requests to make and review\nchanges and versioning changes. GitOps also has operational benefits such as\n[preventing configuration\ndrift](https://about.gitlab.com/topics/gitops/#cicd) and helps engineers\naudit what changes led to certain outcomes with deployments.\n\n\n## Benefits and limitations of git in GitOps workflows\n\n\nWhile git is an essential piece of GitOps workflows, git repositories were\nnot designed to be deployed by GitOps controllers. Git does provide the\nability for engineers to collaborate on infrastructure changes and audit\nthese changes later on, but controllers do not need to download an entire\ngit repository for a successful deployment. GitOps controllers simply need\nthe infrastructure defined for a particular environment.\n\n\nAdditionally, an important piece of the deployment process is to [sign and\nverify\ndeployments](https://docs.sigstore.dev/about/overview/#why-cryptographic-signing)\nto assure deployment changes to an environment are coming from a trusted\nsource. While git commits can be signed and verified by GitOps controllers,\ncommits may also capture other details not related to the deployment itself\n(e.g., documentation changes, updates to other environments, and git\nrepository restructuring) or not enough of the deployment picture as a\ndeployment may consist of multiple commits. This again feels like a case\nthis git feature wasn’t designed for.\n\n\nAnother challenging aspect of git in GitOps workflows is that it can\nsometimes lead to more automation than expected. Soon after merging a change\nto the watched branch, it will be deployed. There are no controls in the\nprocess outside of git. How can you make sure that nothing gets deployed on\na Friday late afternoon? What if teams responsible for deployment do not\nhave permissions to merge changes in certain GitLab projects? Using OCI\nimages adds a pipeline into the process, including all the delivery control\nfeatures, like [approvals or deploy\nfreezes](https://docs.gitlab.com/ee/ci/environments/protected_environments.html).\n\n\n## OCI images\n\n\nThe [Open Container Initiative](https://opencontainers.org/) has helped to\ndefine standards around container formats. While most engineers are familiar\nwith building Dockerfiles into container images, many may not be as familiar\nwith storing Kubernetes manifests in a container registry. Because [GitLab’s\nContainer\nRegistry](https://docs.gitlab.com/ee/user/packages/container_registry/) is\nOCI compliant, it allows for users to push Kubernetes manifests for a\nparticular environment to a container registry. GitOps controllers, such as\n[Flux\nCD](https://about.gitlab.com/blog/why-did-we-choose-to-integrate-fluxcd-with-gitlab/),\ncan use the manifests stored in this OCI artifact instead of needing to\nclone an entire git repository.\n\n\nOften in GitOps workflows, a git repository can include the infrastructure\ndefinitions for all environments that a microservice will be deployed to. By\npackaging the Kubernetes manifests for only a specific environment, Flux CD\ncan download the minimum files needed to carry out a deployment to a\nspecific environment.\n\n\n### Security benefits of using OCI artifacts\n\n\nAs mentioned previously, signing and verifying the artifacts to be deployed\nto an environment adds an additional layer of security for software\nprojects. After Kubernetes manifests are pushed to a container registry, a\ntool like [Sigstore\nCosign](https://docs.sigstore.dev/quickstart/quickstart-cosign/) can be used\nto sign the OCI image with a private key that can be securely stored in a\nGitLab project as a [CI/CD\nvariable](https://docs.gitlab.com/ee/ci/variables/). Flux CD can then use a\npublic key stored on a Kubernetes cluster to verify that a deployment is\ncoming from a trusted source.\n\n\n## Using GitLab to push and sign OCI images\n\n\nGitLab offers many features that help simplify the process of packaging,\nsigning, and deploying OCI images. A common way to structure GitLab projects\nwith GitOps workflows is to have separate GitLab projects for microservices’\ncode and a single infrastructure repository for all microservices. If an\napplication is composed of `n` microservices, this would require having `n\n+1` GitLab projects for an application.\n\n\nThe artifact produced by a code project is usually a container image that\nwill be used to package the application. The infrastructure or delivery\nproject will contain the Kubernetes manifests defining all the resources\nrequired to scale and serve traffic to each microservice. The artifact\nproduced by this project is usually an OCI image used to deploy the\napplication and other manifests to Kubernetes.\n\n\nIn this setup, separation of environments is handled by defining Kubernetes\nmanifests in separate folders. These folders represent environments (e.g.,\ndevelopment, staging, and production) that will host the application. When\nchanges are made to the code project and a new container image is pushed,\nall that needs to be done to deploy these changes via GitLab’s integration\nwith Flux CD is to edit the manifests under the environment folder to\ninclude the new image reference and open a merge request. Once that merge\nrequest is reviewed, approved, and merged, the delivery project’s CI/CD job\nwill push a new OCI image that Flux CD will pick up and deploy to the new\nenvironment.\n\n\n![OCI images - flow\nchart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097611/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097611046.png)\n\n\nSigning an OCI image is as simple as including Cosign in your project’s\nCI/CD job. You can simply generate a new public and private key with Cosign\nby running the commands below locally. Just make sure to log in to your\nGitLab instance with the [glab\nCLI](https://gitlab.com/gitlab-org/cli/#installation) and replace the\n[`PROJECT_ID`] for the Cosign command with your [delivery project’s\nID](https://docs.gitlab.com/ee/user/project/working_with_projects.html#access-a-project-by-using-the-project-id).\n\n\n```\n\nglab auth login\n\ncosign generate-key-pair gitlab://[PROJECT_ID]\n\n```\n\n\nOnce the cosign command runs successfully, you can see the Cosign keys added\nto your project under the CI/CD variables section under the key names\n`COSIGN_PUBLIC_KEY` and `COSIGN_PRIVATE_KEY`.\n\n\n### Example CI/CD job\n\n\nA GitLab CI/CD job for pushing an OCI image will look something like the\nfollowing:\n\n\n```yaml\n\nfrontend-deploy:\n  rules:\n  - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    changes:\n      paths:\n      - manifests/dev/frontend-dev.yaml\n  trigger:\n    include:\n      - component: gitlab.com/components/fluxcd/oci-artifact@0.3.1\n        inputs:\n          version: 0.3.1\n          kubernetes_agent_reference: gitlab-da/projects/tanuki-bank/flux-config:dev\n          registry_image_url: \"oci://$CI_REGISTRY_IMAGE/frontend\"\n          image_tag: dev\n          manifest_path: ./manifests/dev/frontend-dev.yaml\n          flux_oci_repo_name: frontend\n          flux_oci_namespace_name: frontend-dev\n          signing_private_key: \"$COSIGN_PRIVATE_KEY\"\n```\n\n\nThe [GitLab CI/CD\nCatalog](https://about.gitlab.com/blog/ci-cd-catalog-goes-ga-no-more-building-pipelines-from-scratch/)\noffers a GitLab-maintained [CI/CD component for working with OCI artifacts\nand Flux CD](https://gitlab.com/explore/catalog/components/fluxcd). This\ncomponent allows development teams to push Kubernetes manifests as OCI\nimages to GitLab’s Container Registry or an external container registry,\nsign the OCI image using Cosign, and immediately reconcile the newly pushed\nimage via Flux CD.\n\n\nIn the example above, the Flux CD `component` is included in a\n`.gitlab-ci.yml` file of a GitLab project. Using the component’s `inputs`,\nusers can define what registry to push the image to (i.e.,\n`registry_image_url` and `image tag`), the file path to Kubernetes manifests\nthat will be pushed (i.e., `manifest_path`), the cosign private key used to\nsign images (i.e., `signing_private_key`), and the Kubernetes namespace and\nFlux CD\n[OCIRepository](https://fluxcd.io/flux/components/source/ocirepositories/)\nname needed to sync updates to an environment (i.e.,\n`flux_oci_namespace_name` and `flux_oci_repo_name`).\n\n\nThe `kubernetes_agent_reference` allows GitLab CI/CD jobs to inherit the\n`kubeconfig` needed to access a Kubernetes cluster without needing to store\na `kubeconfig` CI/CD variable in each GitLab project. By setting up the\n[GitLab agent for\nKubernetes](https://docs.gitlab.com/ee/user/clusters/agent/), you can\nconfigure all GitLab projects’ CI/CD jobs in a [GitLab\ngroup](https://docs.gitlab.com/ee/user/group/) to inherit permissions to\ndeploy to the Kubernetes cluster.\n\n\nThe agent for Kubernetes context is typically configured wherever you\nconfigure the GitLab Agent for Kubernetes in your GitLab group. It is\ntypically recommended that this be done in the project where Flux CD is\nmanaged. More information on configuring the agent for CI/CD access can be\nfound in our [CI/CD workflow\ndocumentation](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html).\n\n\nThe variables `$COSIGN_PRIVATE_KEY`, `$FLUX_OCI_REPO_NAME`, and\n`$FRONTEND_DEV_NAMESPACE` are values stored as CI/CD variables to easily\naccess and mask these sensitive pieces of data in CI/CD logs. The\n`$CI_REGISTRY_IMAGE` is a variable that GitLab jobs have available by\ndefault that specifies the GitLab project’s container registry.\n\n\n### Deploy OCI images\n\n\nUsing [Flux CD with your GitLab\nprojects](https://docs.gitlab.com/ee/user/clusters/agent/gitops/flux_tutorial.html),\nyou can automate deployments and signing verification for your\nmicroservice’s environments. Once Flux CD is configured to sync from a\nGitLab project, you could add the following Kubernetes [custom resource\ndefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)\nto your project to sync your pushed OCI image.\n\n\n```yaml\n\napiVersion: v1\n\nkind: Namespace\n\nmetadata:\n  name: frontend-dev\n  labels:\n    name: frontend-dev\n---\n\napiVersion: bitnami.com/v1alpha1\n\nkind: SealedSecret\n\nmetadata:\n  name: cosign-public-key\n  namespace: frontend-dev\nspec:\n  encryptedData:\n    cosign.pub: AgAKgLf4VbVzJOmr6++k81LlFayx88AELaUQFNOaXmBF4G+fBfBYeABl0skNvMAa1UrPVNSfMIHgFoYHoO96g576a+epk6V6glOI+++XvYbfsygof3GGxe0nL5Qh2b3ge0fNpyd0kTPSjTj0YUhRhKtMGMRSRw1jrwhNcGxCHK+Byibs52v8Np49KsIkeZKbzLdgYABkrv+k0j7hQM+jR180NpG+2UiRvaXpPuogxkbj61FEqWGrJHk8IVyfl3eh+YhoXxOHGDqko6SUC+bUZPDBlU6yKegO0/8Zq3hwulrSEsEjzRZNK+RFVMOLWWuC6h+WGpYhAMcsZPwjjJ/y29KLNa/YeqkN/cdk488QyEFc6ehCxzhH67HxIn2PDa+KkEOTv2TuycGF+Q00jKIizXF+IwLx/oRb3pTCF0AoAY8D8N3Ey+KfkOjsBON7gGID8GbQiJqX2IgIZxFMk0JRzxbRKOEqn+guLd5Shj7CD1a1Mkk0DxBdbqrGv2XNYUaFPI7xd3rZXUJZlnv+fsmwswsiGWRuXwim45HScWzQnfgLAe7tv3spVEGeaO5apl6d89uN21PBQnfE/zyugB//7ZW9tSp6+CSMyc5HynxI8diafqiwKPgvzLmVWRnkvxJijoXicRr3sCo5RudZPSlnjfd7CKdhwEVvLl7dRR4e/XBMdxCzk1p52Pl+3/kJR+LJii5+iwOpYrpVltSZdzc/3qRd19yMpc9PWpXYi7HxTb24EOQ25i21eDJY1ceplDN6bRtop2quzkjlwVeE2i4cEsX/YG8QBtQbop/3fjiAjKaED3QH3Ul0PECS9ARTScSkcOL3I00Xpp8DyD+xH0/i9wCBRDmH3yKX18C8VrMq02ALSnlP7WCVVjCPzubqKx2LPZRxK9EG0fylwv/vWQzTUUwfbPQZsd4c75bSTsTvxqp/UcFaXA==\n  template:\n    metadata:\n      name: cosign-public-key\n      namespace: frontend-dev\n---\n\napiVersion: source.toolkit.fluxcd.io/v1beta2\n\nkind: OCIRepository\n\nmetadata:\n    name: frontend\n    namespace: frontend-dev\nspec:\n    interval: 1m\n    url: oci://registry.gitlab.com/gitlab-da/projects/tanuki-bank/tanuki-bank-delivery/frontend\n    ref:\n        tag: dev\n    verify:\n      provider: cosign\n      secretRef:\n        name: cosign-public-key\n---\n\napiVersion: kustomize.toolkit.fluxcd.io/v1\n\nkind: Kustomization\n\nmetadata:\n    name: frontend\n    namespace: frontend-dev\nspec:\n    interval: 1m\n    targetNamespace: frontend-dev\n    path: \".\"\n    sourceRef:\n        kind: OCIRepository\n        name: frontend\n    prune: true\n```\n\n\nThe\n[`Kustomization`](https://fluxcd.io/flux/components/kustomize/kustomizations/)\nresource allows for further customization of Kubernetes manifests and also\nspecifies which namespace to deploy resources to. The `OCIRepository`\nresource for Flux CD allows users to specify the OCI image repository\nreference and tag to regularly sync from. Additionally, you will notice the\n`verify.provider` and `verify.secretRef` properties. These fields allow you\nto verify that the OCI image deployed to the cluster was signed by the\ncorresponding Cosign private key used in the earlier CI/CD job.\n\n\nThe public key needs to be stored in a [Kubernetes\nsecret](https://kubernetes.io/docs/concepts/configuration/secret/) that will\nneed to be present in the same namespace as the `OCIRepository` resource. To\nhave this secret managed by Flux CD and not store the secret in plain text,\nyou can consider using\n[SealedSecrets](https://fluxcd.io/flux/guides/sealed-secrets/) to encrypt\nthe value and have it be decrypted cluster side by a controller.\n\n\nFor a simpler approach not requiring SealedSecrets, you can [deploy the\nsecret via a GitLab\nCI/CD](https://docs.gitlab.com/ee/user/clusters/agent/getting_started_deployments.html)\njob using the [`kubectl\nCLI`](https://kubernetes.io/docs/reference/kubectl/). In the non-sealed\nsecret approach, you would simply remove the SealedSecret included above and\nrun the job to deploy the public key secret before running the job to push\nthe new OCI image. This will make sure the secret is stored securely in\nGitLab and make sure the secret can be accessed on the cluster by the\nOCIRepository. While this approach is a bit simpler, just note this is not a\nsuitable approach for managing secrets in production.\n\n\n## The benefits of OCI, GitLab, and GitOps\n\n\nOCI artifacts allow for GitOps teams to take deployments even further with\nadded security benefits and allowing for deployments to be minimal. Users\nstill gain all the benefits offered by git as far as having a source of\ntruth for infrastructure and collaborating on projects. OCI images add a\npackaging approach that improves the deployment aspect of GitOps.\n\n\nGitLab continues to learn from our customers and the cloud native community\non building experiences that help simplify GitOps workflows. To get started\nusing some of the features mentioned in this blog, you can sign up for a\n[free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/). We would also love to hear\nfrom users about their experiences with these tools, and you can provide\nfeedback in the [community\nforum](https://forum.gitlab.com/t/oci-images-as-source-of-truth-for-gitops-with-gitlab/120965).\n",[108,743,1241,549,742,9],{"slug":2936,"featured":6,"template":700},"how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery","content:en-us:blog:how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery.yml","How To Use Oci Images As The Source Of Truth For Continuous Delivery","en-us/blog/how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery.yml","en-us/blog/how-to-use-oci-images-as-the-source-of-truth-for-continuous-delivery",{"_path":2942,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2943,"content":2948,"config":2954,"_id":2956,"_type":13,"title":2957,"_source":15,"_file":2958,"_stem":2959,"_extension":18},"/en-us/blog/how-we-diagnosed-and-resolved-redis-latency-spikes",{"title":2944,"description":2945,"ogTitle":2944,"ogDescription":2945,"noIndex":6,"ogImage":1358,"ogUrl":2946,"ogSiteName":685,"ogType":686,"canonicalUrls":2946,"schema":2947},"How we diagnosed and resolved Redis latency spikes with BPF and other tools","How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle.","https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How we diagnosed and resolved Redis latency spikes with BPF and other tools\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Matt Smiley\"}],\n        \"datePublished\": \"2022-11-28\",\n      }",{"title":2944,"description":2945,"authors":2949,"heroImage":1358,"date":2951,"body":2952,"category":785,"tags":2953},[2950],"Matt Smiley","2022-11-28","If you enjoy performance engineering and peeling back abstraction layers to\nask underlying subsystems to explain themselves, this article’s for you. The\ncontext is a chronic Redis latency problem, and you are about to tour a\npractical example of using BPF and profiling tools in concert with standard\nmetrics to reveal unintuitive behaviors of a complex system.\n\n\nBeyond the tools and techniques, we also use an iterative hypothesis-testing\napproach to compose a behavior model of the system dynamics. This model\ntells us what factors influence the problem's severity and triggering\nconditions.\n\n\nUltimately, we find the root cause, and its remedy is delightfully boring\nand effective. We uncover a three-phase cycle involving two distinct\nsaturation points and a simple fix to break that cycle. Along the way, we\ninspect aspects of the system’s behavior using stack sampling profiles, heat\nmaps and flamegraphs, experimental tuning, source and binary analysis,\ninstruction-level BPF instrumentation, and targeted latency injection under\nspecific entry and exit conditions.\n\n\nIf you are short on time, the takeaways are summarized at the end. But the\njourney is the fun part, so let's dig in!\n\n\n## Introducing the problem: Chronic latency \n\n\nGitLab makes extensive use of Redis, and, on GitLab.com SaaS, we use\n[separate Redis\nclusters](/handbook/engineering/infrastructure/production/architecture/#redis-architecture)\nfor certain functions. This tale concerns a Redis instance acting\nexclusively as a least recently used (LRU) cache.\n\n\nThis cache had a chronic latency problem that started occurring\nintermittently over two years ago and in recent months had become\nsignificantly worse: Every few minutes, it suffered from bursts of very high\nlatency and corresponding throughput drop, eating into its Service Level\nObjective (SLO). These latency spikes impacted user-facing response times\nand [burned error\nbudgets](https://gitlab.com/gitlab-org/gitlab/-/issues/360578#note_966597336)\nfor dependent features, and this is what we aimed to solve.\n\n\n**Graph:** Spikes in the rate of extremely slow (1 second) Redis requests,\neach corresponding to an eviction burst\n\n\n![Graph showing spikes in the slow request rate every few\nminutes](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/00_redis_slow_request_rate_spikes_during_each_eviction_burst.png)\n\n\nIn prior work, we had already completed several mitigating optimizations.\nThese sufficed for a while, but organic growth had resurfaced this as an\nimportant [long-term scaling\nproblem](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#why-is-it-important-to-get-to-the-root-of-the-latency-spikes).\nWe had also already ruled out externally triggered causes, such as request\nfloods, connection rate spikes, host-level resource contention, etc. These\nlatency spikes were consistently associated with memory usage reaching the\neviction threshold (`maxmemory`), not by changes in client traffic patterns\nor other processes competing with Redis for CPU time, memory bandwidth, or\nnetwork I/O.\n\n\nWe [initially\nthought](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1567)\nthat Redis 6.2’s new [eviction throttling\nmechanism](https://github.com/redis/redis/pull/7653) might alleviate our\neviction burst overhead. It did not. That mechanism solves a different\nproblem: It prevents a stall condition where a single call to\n`performEvictions` could run arbitrarily long. In contrast, during this\nanalysis we\n[discovered](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_977816216)\nthat our problem (both before and after upgrading Redis) was related to\nnumerous calls collectively reducing Redis throughput, rather than a few\nextremely slow calls causing a complete stall.\n\n\nTo discover our bottleneck and its potential solutions, we needed to\ninvestigate Redis’s behavior during our workload’s eviction bursts.\n\n\n## A little background on Redis evictions\n\n\nAt the time, our cache was oversubscribed, trying to hold more cache keys\nthan the [configured `maxmemory`\nthreshold](https://redis.io/docs/reference/eviction/) could hold, so\nevictions from the LRU cache were expected. But the dense concentration of\nthat eviction overhead was surprising and troubling.\n\n\nRedis is essentially single-threaded. With a few exceptions, the “main”\nthread does almost all tasks serially, including handling client requests\nand evictions, among other things. Spending more time on X means there is\nless remaining time to do Y, so think about queuing behavior as the story\nunfolds.\n\n\nWhenever Redis reaches its `maxmemory` threshold, it frees memory by\nevicting some keys, aiming to do just enough evictions to get back under\n`maxmemory`. However, contrary to expectation, the metrics for memory usage\nand eviction rate (shown below) indicated that instead of a continuous\nsteady eviction rate, there were abrupt burst events that freed much more\nmemory than expected. After each eviction burst, no evictions occurred until\nmemory usage climbed back up to the `maxmemory` threshold again.\n\n\n**Graph:** Redis memory usage drops by 300-500 MB during each eviction\nburst:\n\n\n![Memory usage repeatedly rises gradually to 64 GB and then abruptly\ndrops](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/01_redis_memory_usage_dips_during_eviction_bursts.png)\n\n\n**Graph:** Key eviction spikes match the timing and size of the memory usage\ndips shown above\n\n\n![Eviction counter shows a large spike each time the previous graph showed a\nlarge memory usage\ndrop](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/02_redis_eviction_bursts.png)\n\n\nThis apparent excess of evictions became the central mystery. Initially, we\nthought answering that question might reveal a way to smooth the eviction\nrate, spreading out the overhead and avoiding the latency spikes. Instead,\nwe discovered that these bursts are an interaction effect that we need to\navoid, but more on that later.\n\n\n## Eviction bursts cause CPU saturation\n\n\nAs shown above, we had found that these latency spikes correlated perfectly\nwith large spikes in the cache’s eviction rate, but we did not yet\nunderstand why the evictions were concentrated into bursts that last a few\nseconds and occur every few minutes.\n\n\nAs a first step, we wanted to verify a causal relationship between eviction\nbursts and latency spikes.\n\n\nTo test this, we used [`perf`](https://www.brendangregg.com/perf.html) to\nrun a CPU sampling profile on the Redis main thread. Then we applied a\nfilter to split that profile, isolating the samples where it was calling the\n[`performEvictions`\nfunction](https://github.com/redis/redis/blob/6.2.6/src/evict.c#L512). Using\n[`flamescope`](https://github.com/Netflix/flamescope), we can visualize the\nprofile’s CPU usage as a [subsecond offset heat\nmap](https://www.brendangregg.com/HeatMaps/subsecondoffset.html), where each\nsecond on the X axis is folded into a column of 20 msec buckets along the Y\naxis. This visualization style highlights sub-second activity patterns.\nComparing these two heat maps confirmed that during an eviction burst,\n`performEvictions` is starving all other main thread code paths for CPU\ntime.\n\n\n**Graph:** Redis main thread CPU time, excluding calls to `performEvictions`\n\n\n![Heat map shows one large gap and two small gaps in an otherwise uniform\npattern of 70 percent to 80 percent CPU\nusage](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/03_heat_map_of_redis_main_thread_during_eviction_burst__excluding_performEvictions.png)\n\n\n**Graph:** Remainder of the same profile, showing only the calls to\n`performEvictions`\n\n\n![This heat map shows the gaps in the previous heap map were CPU time spent\nperforming\nevictions](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/04_heat_map_of_redis_main_thread_during_eviction_burst__only_performEvictions.png)\n\n\nThese results confirm that eviction bursts are causing CPU starvation on the\nmain thread, which acts as a throughput bottleneck and increases Redis’s\nresponse time latency.  These CPU utilization bursts typically lasted a few\nseconds, so they were too short-lived to trigger alerts but were still user\nimpacting.\n\n\nFor context, the following flamegraph shows where `performEvictions` spends\nits CPU time. There are a few interesting things here, but the most\nimportant takeaways are:\n\n* It gets called synchronously by `processCommand` (which handles all client\nrequests).\n\n* It handles many of its own deletes. Despite its name, the `dbAsyncDelete`\nfunction only delegates deletes to a helper thread under certain conditions\nwhich turn out to be rare for this workload.\n\n\n![Flamegraph of calls to function performEvictions, as described\nabove](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/05_flamegraph_of_redis_main_thread_during_eviction_burst__only_performEvictions.png)\n\n\nFor more details on this analysis, see the [walkthrough and\nmethodology](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_854745083).\n\n\n## How fast are individual calls to `performEvictions`?\n\n\nEach incoming request to Redis is handled by a call to `processCommand`, and\nit always concludes by calling the `performEvictions` function. That call to\n`performEvictions` is frequently a no-op, returning immediately after\nchecking that the `maxmemory` threshold has not been breached. But when the\nthreshold is exceeded, it will continue evicting keys until it either\nreaches its `mem_tofree` goal or exceeds its configured time limit per call.\n\n\nThe CPU heat maps shown earlier proved that `performEvictions` calls were\ncollectively consuming a large majority of CPU time for up to several\nseconds.\n\n\nTo complement that, we also measured the wall clock time of individual\ncalls.\n\n\nUsing the `funclatency` CLI tool (part of the [BCC suite of BPF\ntools](https://github.com/iovisor/bcc)), we measured call duration by\ninstrumenting entry and exit from the `performEvictions` function and\naggregated those measurements into a\n[histogram](https://en.wikipedia.org/wiki/Histogram) at 1-second intervals.\nWhen no evictions were occurring, the calls were consistently low latency\n(4-7 usecs/call). This is the no-op case described above (including 2.5\nusecs/call of instrumentation overhead). But during an eviction burst, the\nresults shift to a bimodal distribution, including a combination of the fast\nno-op calls along with much slower calls that are actively performing\nevictions:\n\n\n```\n\n$ sudo funclatency-bpfcc --microseconds --timestamp --interval 1 --duration\n600 --pid $( pgrep -o redis-server )\n'/opt/gitlab/embedded/bin/redis-server:performEvictions'\n\n...\n\n23:54:03\n     usecs               : count     distribution\n         0 -> 1          : 0        |                                        |\n         2 -> 3          : 576      |************                            |\n         4 -> 7          : 1896     |****************************************|\n         8 -> 15         : 392      |********                                |\n        16 -> 31         : 84       |*                                       |\n        32 -> 63         : 62       |*                                       |\n        64 -> 127        : 94       |*                                       |\n       128 -> 255        : 182      |***                                     |\n       256 -> 511        : 826      |*****************                       |\n       512 -> 1023       : 750      |***************                         |\n```\n\n\nThis measurement also directly confirmed and quantified the throughput drop\nin Redis requests handled per second: The call rate to `performEvictions`\n(and hence to `processCommand`) dropped to 20% of its norm from before the\nevictions began, from 25K to 5K calls per second.\n\n\nThis has a huge impact on clients: New requests are arriving at 5x the rate\nthey are being completed. And crucially, we will see soon that this\nasymmetry is what drives the eviction burst.\n\n\nFor more details on this analysis, see the [safety\ncheck](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857869826)\nfor instrumentation overhead and the [results\nwalkthrough](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857907521).\nAnd for more general reference, the BPF instrumentation overhead estimate is\nbased on these [benchmark\nresults](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1383).\n\n\n## Experiment: Can tuning mitigate eviction-driven CPU saturation?\n\n\nThe analyses so far had shown that evictions were severely starving the\nRedis main thread for CPU time. There were still important unanswered\nquestions (which we will return to shortly), but this was already enough\ninfo to [suggest some\nexperiments](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_859236777)\nto test potential mitigations:\n\n* Can we spread out the eviction overhead so it takes longer to reach its\ngoal but consumes a smaller percentage of the main thread’s time?\n\n* Are evictions freeing more memory than expected due to scheduling a lot of\nkeys to be asynchronously deleted by the [lazyfree\nmechanism](https://github.com/redis/redis/blob/6.2.6/redis.conf#L1079)?\nLazyfree is an optional feature that lets the Redis main thread [delegate to\nan async helper\nthread](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_859236777)\nthe expensive task of deleting keys that have more than 64 elements. These\nasync evictions do not count immediately towards the eviction loop’s memory\ngoal, so if many keys qualify for lazyfree, this could potentially drive\nmany extra iterations of the eviction loop.\n\n\nThe\n[answers](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7172#note_971197943)\nto both turned out to be no:\n\n* Reducing `maxmemory-eviction-tenacity` to its minimum setting still did\nnot make `performEvictions` cheap enough to avoid accumulating a request\nbacklog. It did increase response rate, but arrival rate still far exceeded\nit, so this was not an effective mitigation.\n\n* Disabling `lazyfree-lazy-eviction` did not prevent the eviction burst from\ndropping memory usage far below `maxmemory`. Those lazyfrees represent a\nsmall percentage of reclaimed memory. This rules out one of the potential\nexplanations for the mystery of excessive memory being freed.\n\n\nHaving ruled out two potential mitigations and one candidate hypothesis, at\nthis point we return to the pivotal question: Why are several hundred extra\nmegabytes of memory being freed by the end of each eviction burst?\n\n\n## Why do evictions occur in bursts and free too much memory?\n\n\nEach round of eviction aims to free just barely enough memory to get back\nunder the `maxmemory` threshold.\n\n\nWith a steady rate of demand for new memory allocations, the eviction rate\nshould be similarly steady. The rate of arriving cache writes does appear to\nbe steady. So why are evictions happening in dense bursts, rather than\nsmoothly? And why do they reduce memory usage on a scale of hundreds of\nmegabytes rather than hundreds of bytes?\n\n\nSome potential explanations to explore:\n\n* Do evictions only end when a large key gets evicted, spontaneously freeing\nenough memory to skip evictions for a while? No, the memory usage drop is\nfar bigger than the largest keys in the dataset.\n\n* Do deferred lazyfree evictions cause the eviction loop to overshoot its\ngoal, freeing more memory than intended? No, the above experiment disproved\nthis hypothesis.\n\n* Is something causing the eviction loop to sometimes calculate an\nunexpectedly large value for its `mem_tofree` goal? We explore this next.\nThe answer is no, but checking it led to a new insight.\n\n* Is a feedback loop causing evictions to become somehow self-amplifying? If\nso, what conditions lead to entering and leaving this state? This turned out\nto be correct.\n\n\nThese were all plausible and testable hypotheses, and each would point\ntowards a different solution to the eviction-driven latency problem.\n\n\nThe first two hypotheses we have already eliminated.\n\n\nTo test the next two, we built custom BPF instrumentation to peek at the\ncalculation of `mem_tofree` at the start of each call to `performEvictions`.\n\n\n## Observing the `mem_tofree` calculation with `bpftrace`\n\n\nThis part of the investigation was a personal favorite and led to a critical\nrealization about the nature of the problem.\n\n\nAs noted above, our two remaining hypotheses were:\n\n* an unexpectedly large `mem_tofree` goal\n\n* a self-amplifying feedback loop\n\n\nTo differentiate between them, we used\n[`bpftrace`](https://github.com/iovisor/bpftrace) to instrument the\ncalculation of `mem_tofree`, looking at its input variables and results.\n\n\nThis set of measurements directly tests the following:\n\n* Does each call to `performEvictions` aim to free a small amount of memory\n-- perhaps roughly the size of an average cache entry? If `mem_tofree` ever\napproaches hundreds of megabytes, that would confirm the first hypothesis\nand reveal what part of the calculation was causing that large value.\nOtherwise, it rules out the first hypothesis and makes the feedback loop\nhypothesis more likely.\n\n* Does the replication buffer size significantly influence `mem_tofree` as a\nfeedback mechanism? Each eviction adds to this buffer, just like normal\nwrites do. If this buffer grows large (possibly partly due to evictions) and\nthen abruptly shrinks (due to the peer consuming it), that would cause a\nspontaneous large drop in memory usage, ending evictions and instantly\nreducing memory usage. This is one potential way for evictions to drive a\nfeedback loop.\n\n\nTo peek at the values of the `mem_tofree` calculation\n([script](https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt)),\nwe needed to isolate the [correct call from\n`performEvictions`](https://github.com/redis/redis/blob/6.2.6/src/evict.c#L523)\nto the\n[`getMaxmemoryState`](https://github.com/redis/redis/blob/6.2.6/src/evict.c#L374-L407)\nfunction and reverse engineer its assembly to find the right instruction and\nregister to instrument for each of the source code level variables that we\nwanted to capture. From that data we generate histograms for each of the\nfollowing variables:\n\n\n```\n\nmem_reported = zmalloc_used_memory()        // All used memory tracked by\njemalloc\n\noverhead = freeMemoryGetNotCountedMemory()  // Replication output buffers +\nAOF buffer\n\nmem_used = mem_reported - overhead          // Non-exempt used memory\n\nmem_tofree = mem_used - maxmemory           // Eviction goal\n\n```\n\n\n_Caveat:_ Our [custom BPF\ninstrumentation](https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt)\nis specific to this particular build of the `redis-server` binary, since it\nattaches to virtual addresses that are likely to change the next time Redis\nis compiled. But the approach is able to be generalized. Treat this as a\nconcrete example of using BPF to inspect source code variables in the middle\nof a function call without having to rebuild the binary. Because we are\npeeking at the function’s intermediate state and because the compiler\ninlined this function call, we needed to do binary analysis to find the\ncorrect instrumentation points. In general, peeking at a function’s\narguments or return value is easier and more portable, but in this case it\nwould not suffice.\n\n\nThe results:\n\n* Ruled out the first hypothesis: Each call to `performEvictions` had a\nsmall target value (`mem_tofree` \u003C 2 MB). This means each call to\n`performEvictions` did a small amount of work. Redis’s mysterious rapid drop\nin memory usage cannot have been caused by an abnormally large `mem_tofree`\ntarget evicting a big batch of keys all at once. Instead, there must be many\ncalls collectively driving down memory usage.\n\n* The replication output buffers remained consistently small, ruling out one\nof the potential feedback loop mechanisms.\n\n* Surprisingly, `mem_tofree` was usually 16 KB to 64 KB, which is larger\nthan a typical cache entry. This size discrepancy hints that cache keys may\nnot be the main source of the memory pressure perpetuating the eviction\nburst once it begins.\n\n\nAll of the above results were consistent with the feedback loop hypothesis.\n\n\nIn addition to answering the initial questions, we got a bonus outcome:\nConcurrently measuring both `mem_tofree` and `mem_used` revealed a crucial\nnew fact – _the memory reclaim is a completely distinct phase from the\neviction burst_.\n\n\nReframing the pathology as exhibiting separate phases for evictions versus\nmemory reclaim led to a series of realizations, described in the next\nsection. From that emerged a coherent hypothesis explaining all the observed\nproperties of the pathology.\n\n\nFor more details on this analysis, see [methodology\nnotes](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636),\n[build\nnotes](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982499538)\nsupporting the disassembly of the Redis binary, and [initial\ninterpretations](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_977994182).\n\n\n## Three-phase cycle\n\n\nWith the above results indicating a distinct separation between the\nevictions and the memory reclaim, we can now concisely characterize [three\nphases](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982623949)\nin the cycle of eviction-driven latency spikes.\n\n\n**Graph:** Diagram (not to scale) comparing memory and CPU usage to request\nand response rates during each of the three phases\n\n\n![Diagram summarizes the text that follows, showing CPU and memory saturate\nin Phase 2 until request rate drops to match response rate, after which they\nrecover](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/06_3_phase_cycle_of_eviction_bursts.png)\n\n\nPhase 1: Not saturated (7-15 minutes)\n\n* Memory usage is below `maxmemory`. No evictions occur during this phase.\n\n* Memory usage grows organically until reaching `maxmemory`, which starts\nthe next phase.\n\n\nPhase 2: Saturated memory and CPU (6-8 seconds)\n\n* When memory usage reaches `maxmemory`, evictions begin.\n\n* Evictions occur only during this phase, and they occur intermittently and\nfrequently.\n\n* Demand for memory frequently exceeds free capacity, repeatedly pushing\nmemory usage above `maxmemory`. Throughout this phase, memory usage\noscillates close to the `maxmemory` threshold, evicting a small amount of\nmemory at a time, just enough to get back under `maxmemory`.\n\n\nPhase 3: Rapid memory reclaim (30-60 seconds)\n\n* No evictions occur during this phase.\n\n* During this phase, something that had been holding a lot of memory starts\nquickly and steadily releasing it.\n\n* Without the overhead of running evictions, CPU time is again spent mostly\non handling requests (starting with the backlog that accumulated during\nPhase 2).\n\n* Memory usage drops rapidly and steadily. By the time this phase ends,\nhundreds of megabytes have been freed. Afterwards, the cycle restarts with\nPhase 1.\n\n\nAt the transition between Phase 2 and Phase 3, evictions abruptly ended\nbecause memory usage stays below the `maxmemory` threshold.\n\n\nReaching that transition point where memory pressure becomes negative\nsignals that whatever was driving the memory demand in Phase 2 has started\nreleasing memory faster than it is consuming it, shrinking the footprint it\nhad accumulated during the previous phase.\n\n\nWhat is this **mystery memory consumer** that bloats its demand during Phase\n2 and frees it during Phase 3?\n\n\n## The mystery revealed\n\n\n[Modeling the phase\ntransitions](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982651298)\ngave us some useful constraints that a viable hypothesis must satisfy. The\nmystery memory consumer must:\n\n* quickly bloat its footprint to hundreds of megabytes on a timescale of\nless than 10 seconds (the duration of Phase 2), under conditions triggered\nby the start of an eviction burst\n\n* quickly release its accumulated excess on a timescale of just tens of\nseconds (the duration of Phase 3), under the conditions immediately\nfollowing an eviction burst\n\n\n**The answer:** The client input/output buffers meet those constraints to be\nthe mystery memory consumer.\n\n\nHere is how that hypothesis plays out:\n\n* During Phase 1 (healthy state), the Redis main thread’s CPU usage is\nalready fairly high. At the start of Phase 2, when evictions begin, the\neviction overhead saturates the main thread’s CPU capacity, quickly dropping\nresponse rate below the incoming request rate.\n\n* This throughput mismatch between arrivals versus responses **is itself the\namplifier** that takes over driving the eviction burst. As the size of that\nrate gap increases, the proportion of time spent doing evictions also\nincreases.\n\n* Accumulating a backlog of requests requires memory, and that backlog\ncontinues to grow until enough clients are stalled that the arrival rate\ndrops to match the response rate. As clients stall, the arrival rate falls,\nand with it the memory pressure, eviction rate, and CPU overhead begin to\nreduce.\n\n* At the equilibrium point when arrival rate falls to match response rate,\nmemory demand is satisfied and evictions stop (ending Phase 2). Without the\neviction overhead, more CPU time is available to process the backlog, so\nresponse rate increases above request arrival rate. This recovery phase\nsteadily consumes the request backlog, incrementally freeing memory as it\ngoes (Phase 3).\n\n* Once the backlog is resolved, the arrival and response rates match again.\nCPU usage is back to its Phase 1 norm, and memory usage has temporarily\ndropped in proportion to the max size of Phase 2’s request backlog.\n\n\nWe confirmed this hypothesis via a [latency injection\nexperiment](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_987049036)\nshowing that queuing alone explains the pathology. This outcome supports the\nconclusion that the extra memory demand originates from response rate\nfalling below request arrival rate.\n\n\n## Remedies: How to avoid entering the eviction burst cycle\n\n\nNow that we understand the dynamics of the pathology, we can draw confident\nconclusions about viable solutions.\n\n\nRedis evictions are only self-amplifying when all of the following\nconditions are present:\n\n* **Memory saturation:** Memory usage reaches the `maxmemory` limit, causing\nevictions to start.\n\n* **CPU saturation:** The baseline CPU usage by the Redis main thread’s\nnormal workload is close enough to a whole core that the eviction overhead\npushes it to saturation. This reduces the response rate below request\narrival rate, inducing self-amplification via increased memory demand for\nrequest buffering.\n\n* **Many active clients:** The saturation only lasts as long as request\narrival rate exceeds response rate. Stalled clients no longer contribute to\nthat arrival rate, so the saturation lasts longer and has a greater impact\nif Redis has many active clients still sending requests.\n\n\nViable remedies include:\n\n* Avoid memory saturation by any combination of the following to make peak\nmemory usage less than the `maxmemory` limit:\n  * Reduce cache time to live (TTL)\n  * Increase `maxmemory` (and host memory if needed, but watch out for [`numa_balancing` CPU overhead](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1889) on hosts with multiple NUMA nodes)\n  * Adjust client behavior to avoid writing unnecessary cache entries\n  * Split the cache among multiple instances (sharding or functional partitioning, helps avoid both memory and CPU saturation)\n* Avoid CPU saturation by any combination of the following to make peak CPU\nusage for the workload plus eviction overhead be less than 1 CPU core:\n  * Use the fastest processor available for single-threaded instructions per second\n  * Isolate the redis-server process (particularly its main thread) from any other competing CPU-intensive processes (dedicated host, taskset, cpuset)\n  * Adjust client behavior to avoid unnecessary cache lookups or writes\n  * Split the cache among multiple instances (sharding or functional partitioning, helps avoid both memory and CPU saturation)\n  * Offload work from the Redis main thread (io-threads, lazyfree)\n  * Reduce eviction tenacity (only gives a minor benefit in our experiments)\n\nMore exotic potential remedies could include a new Redis feature. One idea\nis to exempt ephemeral allocations like client buffers from counting towards\nthe `maxmemory` limit, instead applying that limit only to key storage.\nAlternatively, we could limit evictions to only consume at most a\nconfigurable percentage of the main thread’s time, so that most of its time\nis still spent on request throughput rather than eviction overhead.\n\n\nUnfortunately, either of those features would trade one failure mode for\nanother, reducing the risk of eviction-driven CPU saturation while\nincreasing the risk of unbounded memory growth at the process level, which\ncould potentially saturate the host or cgroup and lead to an OOM, or out of\nmemory, kill. That trade-off may not be worthwhile, and in any case it is\nnot currently an option.\n\n\n## Our solution\n\n\nWe had already exhausted the low-hanging fruit for CPU efficiency, so we\nfocused our attention on avoiding memory saturation.\n\n\nTo improve the cache’s memory efficiency, we\n[evaluated](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_990891708)\nwhich types of cache keys were using the most space and how much\n[`IDLETIME`](https://redis.io/commands/object-idletime/) they had accrued\nsince last access. This memory usage profile identified some rarely used\ncache entries (which waste space), helped inform the TTL, or time to live,\ntuning by first focusing on keys with a high idle time, and highlighted some\nuseful potential cutpoints for functionally partitioning the cache.\n\n\nWe\n[decided](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_1014582669)\nto concurrently pursue several cache efficiency improvements and opened an\n[epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/764) for it.\nThe goal was to avoid chronic memory saturation, and the main action items\nwere:\n\n* Iteratively reduce the cache’s [default\nTTL](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1854) from\n2 weeks to 8 hours (helped a lot!)\n\n* Switch to [client-side\ncaching](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_1026821730)\nfor certain cache keys (efficiently avoids spending shared cache space on\nnon-shared cache entries)\n\n* [Partition](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/762) a\nset of cache keys to a separate Redis instance\n\n\nThe TTL reduction was the simplest solution and turned out to be a big win.\nOne of our main concerns with TTL reduction was that the additional cache\nmisses could potentially increase workload on other parts of the\ninfrastructure. Some cache misses are more expensive than others, and our\nmetrics are not granular enough to quantify the cost of cache misses per\ntype of cache entry. This concern is why we applied the TTL adjustment\nincrementally and monitored for SLO violations. Fortunately, our inference\nwas correct: Reducing TTL did not significantly reduce the cache hit rate,\nand the additional cache misses did not cause noticeable impact to\ndownstream subsystems.\n\n\nThe TTL reduction turned out to be sufficient to drop memory usage\nconsistently a little below its saturation point.\n\n\nIncreasing `maxmemory` had initially not been feasible because the original\npeak memory demand (prior to the efficiency improvements) was expected to be\nlarger than the max size of the VMs we use for Redis. However, once we\ndropped memory demand below saturation, then we could confidently [provision\nheadroom](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1868)\nfor future growth and re-enable [saturation\nalerting](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1883).\n\n\n## Results\n\n\nThe following graph shows Redis memory usage transitioning out of its\nchronically saturated state, with annotations describing the milestones when\nlatency spikes ended and when saturation margin became wide enough to be\nconsidered safe:\n\n\n![Redis memory usage stops showing a flat top\nsaturation](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/07_epic_results__memory_saturation_avoided_by_TTL_reductions.png)\n\n\nZooming into the days when we rolled out the TTL adjustments, we can see the\nharmful eviction-driven latency spikes vanish as we drop memory usage below\nits saturation point, exactly as predicted:\n\n\n![Redis memory usage starts as a flat line and then falls below that\nsaturation\nline](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/08_results__redis_memory_usage_stops_saturating.png)\n\n\n![Redis response time spikes stop occurring at the exact point when memory\nstops being\nsaturated](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/09_results__redis_latency_spikes_end.png)\n\n\nThese eviction-driven latency spikes had been the biggest cause of slowess\nin Redis cache.\n\n\nSolving this source of slowness significantly improved the user experience.\nThis 1-year lookback shows only the long-tail portion of the improvement,\nnot even the full benefit.  Each weekday had roughly 2 million Redis\nrequests slower than 1 second, until our fix in mid-August:\n\n\n![Graph of the daily count of Redis cache requests slower than 1 second,\nshowing roughly 2 million slow requests per day on weekdays until\nmid-August, when the TTL adjustments were\napplied](https://about.gitlab.com/images/blogimages/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/10_results__1_year_retrospective_of_slow_redis_requests_per_day.png)\n\n\n## Conclusions\n\n\nWe solved a long-standing latency problem that had been worsening as the\nworkload grew, and we learned a lot along the way. This article focuses\nmostly on the Redis discoveries, since those are general behaviors that some\nof you may encounter in your travels. We also developed some novel tools and\nanalytical methods and uncovered several useful environment-specific facts\nabout our workload, infrastructure, and observability, leading to several\nadditional improvements and proposals not mentioned above.\n\n\nOverall, we made several efficiency improvements and broke the cycle that\nwas driving the pathology. Memory demand now stays well below the saturation\npoint, eliminating the latency spikes that were burning error budgets for\nthe development teams and causing intermittent slowness for users. All\nstakeholders are happy, and we came away with deeper domain knowledge and\nsharper skills!\n\n\n## Key insights summary\n\n\nThe following notes summarize what we learned about Redis eviction behavior\n(current as of version 6.2):\n\n* The same memory budget (`maxmemory`) is shared by key storage and client\nconnection buffers. A spike in demand for client connection buffers counts\ntowards the `maxmemory` limit, in the same way that a spike in key inserts\nor key size would.\n\n* Redis performs evictions in the foreground on its main thread. All time\nspent in `performEvictions` is time not spent handling client requests.\nConsequently, during an eviction burst, Redis has a lower throughput\nceiling.\n\n* If eviction overhead saturates the main thread’s CPU, then response rate\nfalls below request arrival rate. Redis accumulates a request backlog (which\nconsumes memory), and clients experience this as slowness.\n\n* The memory used for pending requests requires more evictions, driving the\neviction burst until enough clients are stalled that arrival rate falls back\nbelow response rate. At that equilibrium point, evictions stop, eviction\noverhead vanishes, Redis rapidly handles its request backlog, and that\nbacklog’s memory gets freed.\n\n* Triggering this cycle requires all of the following:\n  * Redis is configured with a `maxmemory` limit, and its memory demand exceeds that size. This memory saturation causes evictions to begin.\n  * Redis main thread’s CPU utilization is high enough under its normal workload that having to also perform evictions drives it to CPU saturation. This reduces response rate below request rate, causing a growing request backlog and high latency.\n  * Many active clients are connected. The duration of the eviction burst and the size of memory spent on client connection buffers increases proportionally to the number of active clients.\n* Prevent this cycle by avoiding either memory or CPU saturation. In our\ncase, avoiding memory saturation was easier (mainly by reducing cache TTL).\n\n\n## Further reading\n\n\nThe following lists summarize the analytical tools and methods cited in this\narticle. These tools are all highly versatile and any of them can provide a\nmassive level-up when working on performance engineering problems.\n\n\nTools:\n\n* [perf](https://www.brendangregg.com/perf.html) - A Linux performance\nanalysis multitool. In this article, we used `perf` as a sampling profiler,\ncapturing periodic stack traces of the `redis-server` process's main thread\nwhen it is actively running on a CPU.\n\n* [Flamescope](https://github.com/Netflix/flamescope) - A visualization tool\nfor rendering a `perf` profile (and other formats) into an interactive\nsubsecond heat map. This tool invites the user to explore the timeline for\nmicrobursts of activity or inactivity and render flamegraphs of those\ninteresting timespans to explore what code paths were active.\n\n* [BCC](https://github.com/iovisor/bcc) - BCC is a framework for building\nBPF tools, and it ships with many useful tools out of the box. In this\narticle, we used `funclatency` to measure the call durations of a specific\nRedis function and render the results as a histogram.\n\n* [bpftrace](https://github.com/iovisor/bpftrace) - Another BPF framework,\nideal for answering ad-hoc questions about your system's behavior. It uses\nan `awk`-like syntax and is [quick to\nlearn](https://github.com/iovisor/bpftrace#readme). In this article, we\nwrote a [custom `bpftrace`\nscript](https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt)\nfor observing the variables used in computing how much memory to free during\neach round of evictions. This script's instrumentation points are specific\nto our particular build of `redis-server`, but the [approach is able to be\ngeneralized](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636)\nand illustrates how versatile this tool can be.\n\n\nUsage examples:\n\n*\n[Example](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_854745083)\n- Walkthrough of using `perf` and `flamescope` to capture, filter, and\nvisualize the stack sampling CPU profiles of the Redis main thread.\n\n*\n[Example](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857869826)\n- Walkthrough (including safety check) of using `funclatency` to measure the\ndurations of the frequent calls to function `performEvictions`.\n\n*\n[Example](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7172#note_971197943)\n- Experiment for adjusting Redis settings `lazyfree-lazy-eviction` and\n`maxmemory-eviction-tenacity` and observing the results using `perf`,\n`funclatency`, `funcslower`, and the Redis metrics for eviction count and\nmemory usage.\n\n*\n[Example](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636)\n- This is a working example (script included) of using `bpftrace` to observe\nthe values of a function's variables. In this case we inspected the\n`mem_tofree` calculation at the start of `performEvictions`. Also, these\n[companion\nnotes](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982499538)\ndiscuss some build-specific considerations.\n\n*\n[Example](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_987049036)\n- Describes the latency injection experiment (the first of the three ideas).\nThis experiment confirmed that memory demand increases at the predicted rate\nwhen we slow response rate to below request arrival rate, in the same way\nevictions do. This result confirmed the request queuing itself is the source\nof the memory pressure that amplifies the eviction burst once it begins.\n",[744,9,789],{"slug":2955,"featured":6,"template":700},"how-we-diagnosed-and-resolved-redis-latency-spikes","content:en-us:blog:how-we-diagnosed-and-resolved-redis-latency-spikes.yml","How We Diagnosed And Resolved Redis Latency Spikes","en-us/blog/how-we-diagnosed-and-resolved-redis-latency-spikes.yml","en-us/blog/how-we-diagnosed-and-resolved-redis-latency-spikes",{"_path":2961,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2962,"content":2968,"config":2974,"_id":2976,"_type":13,"title":2977,"_source":15,"_file":2978,"_stem":2979,"_extension":18},"/en-us/blog/icymi-key-ai-and-security-insights-from-our-developer-community",{"title":2963,"description":2964,"ogTitle":2963,"ogDescription":2964,"noIndex":6,"ogImage":2965,"ogUrl":2966,"ogSiteName":685,"ogType":686,"canonicalUrls":2966,"schema":2967},"ICYMI: Key AI and security insights from our developer community","Our latest LinkedIn Live highlights the hottest trends in AI, security, DevSecOps, and more. Also get a taste of the GitLab community contributions that are making an impact.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098331/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%286%29_55zMmdJIUpfh5qaPW9dtVA_1750098331584.png","https://about.gitlab.com/blog/icymi-key-ai-and-security-insights-from-our-developer-community","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"ICYMI: Key AI and security insights from our developer community\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fatima Sarah Khalid\"}],\n        \"datePublished\": \"2024-12-05\",\n      }",{"title":2963,"description":2964,"authors":2969,"heroImage":2965,"date":2970,"body":2971,"category":1119,"tags":2972},[1217],"2024-12-05","In our [November LinkedIn Live broadcast](https://www.linkedin.com/feed/update/urn:li:activity:7265408726696697857), we brought together field CTOs, developer advocates, and community leaders to discuss industry trends and showcase features making a difference in developer workflows.\n\nHere are 5 key highlights:\n\n### 1. AI adoption trends from the field\nOur field CTOs shared insights on how organizations are embracing AI across their development workflows. For instance, Field CTO Cherry Han highlighted how financial organizations are thinking beyond individual developer tools.\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1035388263?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Ai Adoption Trends from the Field\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n\u003Cbr>\u003C/br>\nAndrew Hasker, Field CTO for Asia Pacific and Japan, offered valuable perspective on AI adoption.\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1035388277?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"From Field CTOs\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n### 2. Security coverage that makes a difference\n\nStaff Developer Advocate Fernando Diaz demonstrated how GitLab's security scanners cover the complete application lifecycle, showing how easy it is to implement [comprehensive security scanning](https://about.gitlab.com/solutions/security-compliance/) with just a few lines of code.\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1035388297?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Security Coverage\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n### 3. AI-powered language migration made simple\nIn an impressive demonstration, Senior Technical Marketing Manager Cesar Saavedra showed how GitLab Duo can assist in migrating applications between programming languages.\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1036170482?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"AI-Powered Language Migration Made Simple\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n### 4. Making DevSecOps work smarter\n\nDeveloper Advocate Abubakar Siddiq Ango showcased how GitLab's triage features can automate routine tasks.\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1035388290?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Making DEvOps Work Smarter\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n### 5. Community contributions making an impact\n\nDirector of Contributor Success Nick Veenhof shared how community contributions are shaping GitLab's development:\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe src=\"https://player.vimeo.com/video/1035395211?badge=0&autopause=0&player_id=0&app_id=58479\" frameborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture; clipboard-write\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;\" title=\"Community Contributions Making an Impact\">\u003C/iframe>\u003C/div>\u003Cscript src=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n## Watch on-demand\n\n[Watch the complete broadcast recording](https://www.linkedin.com/feed/update/urn:li:activity:7265408726696697857) for step-by-step demonstrations and insights from our experts. Also, be sure to [follow GitLab on LinkedIn](https://www.linkedin.com/company/gitlab-com) to stay up to date on our monthly broadcasts and get insights into our platform, DevSecOps, and software development.\n",[720,830,268,9,2973],"webcast",{"slug":2975,"featured":6,"template":700},"icymi-key-ai-and-security-insights-from-our-developer-community","content:en-us:blog:icymi-key-ai-and-security-insights-from-our-developer-community.yml","Icymi Key Ai And Security Insights From Our Developer Community","en-us/blog/icymi-key-ai-and-security-insights-from-our-developer-community.yml","en-us/blog/icymi-key-ai-and-security-insights-from-our-developer-community",{"_path":2981,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":2982,"content":2988,"config":2992,"_id":2994,"_type":13,"title":2995,"_source":15,"_file":2996,"_stem":2997,"_extension":18},"/en-us/blog/improve-security-auditing-with-gitlab-operational-container-scanning",{"title":2983,"description":2984,"ogTitle":2983,"ogDescription":2984,"noIndex":6,"ogImage":2985,"ogUrl":2986,"ogSiteName":685,"ogType":686,"canonicalUrls":2986,"schema":2987},"Improve security auditing with GitLab Operational Container Scanning","Learn how to conduct container vulnerability scans post-deployment to raise awareness of existing threats and to track resolution of vulnerabilities.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664654/Blog/Hero%20Images/AdobeStock_1172300481.jpg","https://about.gitlab.com/blog/improve-security-auditing-with-gitlab-operational-container-scanning","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Improve security auditing with GitLab Operational Container Scanning\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Daniel Helfand\"}],\n        \"datePublished\": \"2025-01-29\",\n      }",{"title":2983,"description":2984,"authors":2989,"heroImage":2985,"date":1034,"body":2990,"category":830,"tags":2991},[2874],"Conducting security scans is a regular part of any software development\nprocess. Whether scanning source code (e.g., Java, Python, or other\nlanguages), configuration files (e.g., YAML files), or [container\nimages](https://cloudnativenow.com/kubecon-cnc-na-2024/unlocking-the-full-potential-of-container-vulnerability-scans/),\nthese scanning tools help development teams be proactive about understanding\nand addressing security threats. \n\n\nTraditionally, developers run these [security scans as part of CI/CD\npipelines](https://docs.gitlab.com/ee/user/application_security/container_scanning/).\nBy including these scans in CI/CD, every change to a project will be\nreviewed to see if any vulnerabilities are introduced. Understanding\nsecurity concerns during development helps to assure that changes are\naddressed before they are deployed to a live environment, but there are many\nadditional benefits to conducting container vulnerability scans post\ndeployment as well.\n\n\n[GitLab's Operational Container\nScanning](https://docs.gitlab.com/ee/user/clusters/agent/vulnerabilities.html)\nfeature allows DevSecOps practitioners to run container vulnerability scans\nagainst containers running in a Kubernetes environment. The benefits of\nconducting a vulnerability scan on deployed containers include regularly\nscanning the images for new vulnerabilities that are discovered, tracking\nwhich environments certain vulnerabilities are deployed to, and also\ntracking the progress of resolving these vulnerabilities. \n\n\nThe scans can be configured to run on a regular cadence and on containers in\nspecific namespaces on a Kubernetes cluster. The results of these scans are\nthen sent back to GitLab projects to be viewed via the GitLab UI. To show\nexactly how the feature works, the next steps in this article will\ndemonstrate how to apply the Operational Container Scanning feature using a\nGitLab project, sample application, and a Kubernetes cluster. \n\n\n## Prerequisites\n\n\nTo get started, you will need the following:   \n\n* [GitLab Ultimate account](https://about.gitlab.com/free-trial/)   \n\n* Kubernetes cluster that meets [GitLab’s Kubernetes version\nrequirements](https://docs.gitlab.com/ee/user/clusters/agent/#supported-kubernetes-versions-for-gitlab-features)  \n\n* [kubectl CLI](https://kubernetes.io/docs/tasks/tools/#kubectl)\n\n* [helm CLI](https://helm.sh/docs/intro/install/)\n\n\nAdditionally, the walkthrough below will use a [GitLab\nproject](https://gitlab.com/gitlab-da/tutorials/cloud-native/operational-container-scanning-tutorial)\nthat can be forked into a [GitLab\ngroup](https://docs.gitlab.com/ee/user/group/) where you have appropriate\npermissions to carry out the steps that follow. \n\n\n## Deploy a sample application\n\n\nThe first action we will carry out is to deploy a sample application to the\nKubernetes cluster you will use in this tutorial. Before running the\n`kubectl` command to deploy a sample application, take a moment to make sure\nyour `KUBECONFIG` is set to the cluster you would like to use. Once you are\nset up to use your cluster, run the following command:\n\n\n```bash  \n\n$ kubectl apply -f\n\nhttps://gitlab.com/gitlab-da/tutorials/cloud-native/go-web-server/-/raw/main/manifests/go-web-server-manifests.yaml\n\n\nnamespace/go-web-server-dev created  \n\ndeployment.apps/go-web-server created  \n\nservice/go-web-server created  \n\n```\n\n\nWait for all the pods to be running in the `go-web-server-dev` namespace by\nrunning the command below:\n\n\n```bash  \n\n$ kubectl get pods -n go-web-server-dev -w  \n\n```\n\n\nYou should see output similar to what is shown below:\n\n\n```  \n\nNAME                            READY   STATUS    RESTARTS   AGE  \n\ngo-web-server-f6b8767dc-57269   1/1     Running   0          18m  \n\ngo-web-server-f6b8767dc-fkct2   1/1     Running   0          18m  \n\ngo-web-server-f6b8767dc-j4qwg   1/1     Running   0          18m  \n\n```\n\n\nOnce everything is running, you can set up your forked GitLab project to\nconnect to your Kubernetes cluster and configure the Operational Container\nScanning properties. \n\n\n## Connect Kubernetes cluster\n\n\nIn this section, you will learn how to connect a Kubernetes cluster to your\nGitLab project via the [GitLab Agent for\nKubernetes](https://docs.gitlab.com/ee/user/clusters/agent/). By configuring\nand installing the agent on your Kubernetes cluster, you will be able to\nalso configure Operational Container Scanning. \n\n\n### Change the id property for GitLab’s Kubernetes agent\n\n\nIn the forked GitLab project you are using, change the [`id` property in the\nconfig.yaml\nfile](https://gitlab.com/gitlab-da/tutorials/cloud-native/operational-container-scanning-tutorial/-/blob/main/.gitlab/agents/k8s-agent/config.yaml?ref\\_type=heads\\#L5)\nto match the group where you have forked the project. By doing this, you\nwill configure the GitLab Agent for Kubernetes to pass information about\nyour cluster back to your GitLab project. Make sure to commit and push this\nchange back to the main branch of the forked project.\n\n\n### Navigate to Kubernetes clusters page of the project\n\n\nIn the GitLab UI, select the **Operate > Kubernetes clusters** tab of the\nforked project. Click the **Connect a cluster (agent)** button. Add the name\nof the agent to the input box under `Option 2: Create and register an agent\nwith the UI` and then click **Create and register**. In this case, the name\nof the agent is `k8s-agent` since the folder under agents with the\n`config.yaml` file is named `k8s-agent`. Note that this folder can have any\nname that follows [Kubernetes naming\nrestrictions](https://docs.gitlab.com/ee/user/clusters/agent/install/#create-an-agent-configuration-file)\nand that `k8s-agent` is just being used for simplicity.\n\n\n### Install the GitLab Kubernetes agent\n\n\nAfter registering the agent, you will be asked to run a helm command shown\nin the GitLab UI from your command line against your Kubernetes cluster.\nBefore running the command, make sure your `KUBECONFIG` is still connected\nto the same cluster where you deployed the sample application. \n\n\nAfter running the helm command successfully, wait for all pods to be running\nin the `gitlab-agent-k8s-agent` namespace on your cluster. You can wait for\neverything to be running using the following command: \n\n\n```bash  \n\n$ kubectl get pods -n gitlab-agent-k8s-agent -w  \n\n``` \n\n\nYou should see similar output to what is shown below:\n\n\n```  \n\nNAME                                         READY   STATUS    RESTARTS  \nAGE  \n\nk8s-agent-gitlab-agent-v2-6bb676b6bf-v4qml   1/1     Running   0         \n10m  \n\nk8s-agent-gitlab-agent-v2-6bb676b6bf-xt7xh   1/1     Running   0         \n10m  \n\n```\n\n\nOnce the pods are running, your GitLab project should be connected to your\nKubernetes cluster and ready to use the Operational Container Scanning\nfeature. Before proceeding, continue running the `kubectl get pods -n\ngitlab-agent-k8s-agent -w` command to help explain concepts in the next\nsection.\n\n\n## Operational Container Scanning\n\n\nIn addition to the pods for the GitLab agent running in the\n`gitlab-agent-k8s-agent` namespace, there should eventually be another pod\nnamed `trivy-scan-go-web-server-dev`. This pod will start and run on a\nregular cadence and conduct a container vulnerability scan using a tool\nnamed [trivy](https://trivy.dev/latest/) against the `go-web-server-dev`\nnamespace where the sample application deployed earlier is running. \n\n\nThe Operational Container Scanning properties are defined in the\n[`config.yaml`\nfile](https://gitlab.com/gitlab-da/tutorials/cloud-native/operational-container-scanning-tutorial/-/blob/main/.gitlab/agents/k8s-agent/config.yaml?ref_type=heads#L6-L10)\nused to set up the GitLab agent for Kubernetes on your cluster. \n\n\nThe two main properties to define are `cadence`, which specifies how\nfrequently to run the container vulnerability scan, and also the\n`namespaces` property nested under `vulnerability_report`, which defines one\nor more namespaces to conduct the scan on. You can see how this looks in\n`config.yaml` below:\n\n\n```yaml  \n\ncontainer_scanning:  \n  cadence: '*/5 * * * *'  \n  vulnerability_report:  \n    namespaces:  \n      - go-web-server-dev  \n```\n\n\nThe cadence follows a cron format. In this case, `*/5 * * * *` means the\nscan will be run every five minutes, but this can be changed to any amount\nof time (e.g., every 24 hours).  \n\n\nThe vulnerabilities revealed by the scan for containers running in the\n`go-web-server-dev` namespace are sent back to your GitLab project. To see\nthe results, go to the GitLab UI and select your forked project. Select the\n**Secure > Vulnerability report** option for the project and then select the\n**Operational vulnerabilities** tab to view scan results. \n\n\nThe scan results will include information on the severity of the common\nvulnerabilities and exposures (CVEs), along with the name of the image. By\nusing the tag of the image to include the version of the deployed software\nalong with what environment it is deployed to, you can begin to audit what\nknown vulnerabilities exist in your Kubernetes environments and keep track\nof how they are being addressed by engineering teams.\n\n\nWatch this demo for more information:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/2FVQec2J-Ew?si=T6kwPMnPAGwKlkfP\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Share your feedback\n\n\nAdding GitLab’s Operational Container Scanning to your Kubernetes\nenvironments can help development, security, and infrastructure teams have a\nconsistent picture of container security in Kubernetes environments across\nan organization. In addition to GitLab’s CI container scanning capabilities\nand the ability to [scan containers pushed to GitLab’s container\nregistry](https://www.youtube.com/watch?v=Zuk7Axs-CRw), GitLab has solutions\nat every phase of the software development lifecycle to address container\nsecurity concerns.\n\n\nYou can share your feedback on Operational Container Scanning in this [forum\npost](https://forum.gitlab.com/t/operational-container-scanning-feedback/119479),\nwhich we will share with our product and engineering teams supporting this\nfeature. You can get started with Operational Container Scanning by reading\nthe [documentation on the\nfeature](https://docs.gitlab.com/ee/user/clusters/agent/vulnerabilities.html)\nand [starting a free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).\n",[696,851,9,830,694],{"slug":2993,"featured":6,"template":700},"improve-security-auditing-with-gitlab-operational-container-scanning","content:en-us:blog:improve-security-auditing-with-gitlab-operational-container-scanning.yml","Improve Security Auditing With Gitlab Operational Container Scanning","en-us/blog/improve-security-auditing-with-gitlab-operational-container-scanning.yml","en-us/blog/improve-security-auditing-with-gitlab-operational-container-scanning",{"_path":2999,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3000,"content":3006,"config":3012,"_id":3014,"_type":13,"title":3015,"_source":15,"_file":3016,"_stem":3017,"_extension":18},"/en-us/blog/inside-the-improved-ci-logs-management-experience-for-multi-line-commands",{"title":3001,"description":3002,"ogTitle":3001,"ogDescription":3002,"noIndex":6,"ogImage":3003,"ogUrl":3004,"ogSiteName":685,"ogType":686,"canonicalUrls":3004,"schema":3005},"Inside the improved CI logs management experience for multi-line commands","Reviewing log output for CI/CD jobs with multi-line commands is now easier than ever. Find out why, how to configure your pipelines, and what's ahead.\n\n","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099499/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_639935439_3oqldo5Yt5wPonEJYZOLTM_1750099498739.jpg","https://about.gitlab.com/blog/inside-the-improved-ci-logs-management-experience-for-multi-line-commands","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Inside the improved CI logs management experience for multi-line commands\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Romuald Atchadé\"}],\n        \"datePublished\": \"2024-01-25\",\n      }",{"title":3001,"description":3002,"authors":3007,"heroImage":3003,"date":3009,"body":3010,"category":785,"tags":3011},[3008],"Romuald Atchadé","2024-01-25","Improving the GitLab CI/CD log experience for jobs with multi-line commands\nhas been a long-requested feature. With the latest release of GitLab and\nGitLab Runner, it's now easier to work with the log section for jobs with\nmulti-line commands. In this post, we will describe the experience with the\nnew feature, show you how to enable the new log output in your pipelines,\nand discuss key points regarding CI/CD script execution and log output in\nvarious shells, such as Bash and Powershell.\n\n\n## Overview of multi-line commands\n\n\nFirst, it’s helpful to describe what we mean by a CI job with multi-line\ncommands. In GitLab CI the script keyword is used to specify commands to\nexecute for a CI job. In the example below, the build-job has a single\ncommand, a basic echo statement, to execute in the script block. \n\n\n```\n\n## A pipeline with a single line command in the script block for the\nbuild-job\n\n\nbuild-job:\n  stage: build\n  script:\n    - echo \"this is the script to run for the build job\"\n\n```\n\n\nIf you were to run this pipeline, then the log output in the UI would\ndisplay as follows:\n\nLine 17 - GitLab CI automatically generates a log entry for the command that\nyou specify in the script block.\n\nLine 18 - This is the output of the command that was executed.\n\n\n![Ci log management - image\n2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099524/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099524655.png)\n\n\nNow as you can imagine, the script that you define in the script block will\nlikely be more complex than the example provided and could very well span\nmultiple lines in the CI/CD pipeline file. \n\n\n```\n\n## A pipeline with a multi-line command in the script block for the\nbuild-job\n\n\nbuild-job:\n  stage: build\n  script:\n       - |\n         echo \"this is a multi-line command\"  # a simple echo statement\n         ls  \n\n```\n\n\nIf you were to run this pipeline, then the log output in the UI would\ndisplay as follows:\n\n\nLine 17 - As in the previous example, GitLab CI automatically generates a\nlog entry for the command that you specify in the script block. You will\nnotice that line 17 only includes the first command in the script block.\nThis makes it more difficult to debug an issue with script execution as you\nwill need to refer back to the source pipeline file to see exactly what\nscript was executed.\n\n\n![CI log management - image\n3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099525/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099524656.png)\n\n\n## So what’s new?\n\n\nStarting in GitLab 16.7 and GitLab Runner 16.7, you can now enable a feature\nflag titled FF_SCRIPT_SECTIONS, which will add a collapsible output section\nto the CI job log for multi-line command script blocks. This feature flag\nchanges the log output for CI jobs that execute within the Bash shell.\n\n\n![CI log management - image\n4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099525/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099524658.png)\n\n\nLine 17: Unlike the previous examples, the first thing you will notice in\nthe screenshot above is that by default the log entry for the multi-line\ncommand is collapsed by default.\n\n\nSingle-line commands do not display in a collapsible element.\n\n\nFor multi-line scripts the multi-line command is now a collapsible element,\nso now, when you uncollapse the log entry for line 17, then the log will\ndisplay all of the commands that were executed in the script block.\n\n\n![CI log management - image\n1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099525/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099524659.png)\n\n\nThere is also the [`custom collapsible\nsection`](https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections)\nfeature, which in combination with this new multi-command output capability\ndoes provide you additional flexibility for displaying log output in the UI.\nHere is how you can use the two features to change the log output. \n\n\n```\n\n## A pipeline with a multi-line command in the script block for the\nbuild-job\n\n\nvariables:\n  FF_PRINT_POD_EVENTS: \"true\"\n  FF_USE_POWERSHELL_PATH_RESOLVER: \"true\"\n  FF_SCRIPT_SECTIONS: \"true\"\n\ncollapsible_job_multiple:\n  stage: build\n  script:\n    - |\n      echo \"{\n        'test': 'data',\n        'test2': 'data2',\n      }\"\n    - |\n      echo \"{\n        'test': 'data',\n        'test2': 'data2',\n      }\"\n    - echo -e \"\\033[0Ksection_start:`date +%s`:my_first_section\\r\\033[0KHeader of the 1st collapsible section\"\n    - echo 'this line should be hidden when collapsed'\n    - |\n      echo \"{\n        'test': 'data',\n        'test2': 'data2',\n      }\"\n    - echo -e \"\\033[0Ksection_start:`date +%s`:second_section\\r\\033[0KHeader of the 2nd collapsible section\"\n    - echo 'this line should be hidden when collapsed'\n    - echo -e \"\\033[0Ksection_end:`date +%s`:second_section\\r\\033[0K\"\n    - echo -e \"\\033[0Ksection_end:`date +%s`:my_first_section\\r\\033[0K\"\n\n```\n\n\nIf you were to run this pipeline with the FF_SCRIPT_SECTIONS feature flag\nset to false, then the log output would be as depicted in the following\nscreenshot.\n\n\n![CI log management - image\n5](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099524/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099524661.png)\n\n\nBut, if you were to run this pipeline with the FF_SCRIPT_SECTIONS feature\nflag set to true, then the log output would be as depicted in the following\nscreenshot.\n\n\n![CI log management - image\n6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099525/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099524663.png)\n\n\n## What about other shells?\n\n\nAs of the 16.7 release, the collapsible output section in the CI job log for\nmulti-line command script blocks is only visible for CI/CD jobs that are\nexecuted with the Bash shell. CI/CD jobs executed with Powershell is not\ncurrently supported. We plan to add this\n[capability](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/4494)\nin a future release. \n\n\n## What are our future plans?\n\n\nA few features are still needed to improve the CI/CD job log output, and the\n`timestamp` for each log line is one of them. This addition will add missing\nfeatures such as command/section duration.\n\n\n> To learn more about GitLab CI/CD features, refer to the official [CI/CD\ndocumentation](https://docs.gitlab.com/ee/ci/index.html). \n\n\n_Disclaimer: This blog contains information related to upcoming products,\nfeatures, and functionality. It is important to note that the information in\nthis blog post is for informational purposes only. Please do not rely on\nthis information for purchasing or planning purposes. As with all projects,\nthe items mentioned in this blog and linked pages are subject to change or\ndelay. The development, release, and timing of any products, features, or\nfunctionality remain at the sole discretion of GitLab._\n",[787,788,108,9],{"slug":3013,"featured":90,"template":700},"inside-the-improved-ci-logs-management-experience-for-multi-line-commands","content:en-us:blog:inside-the-improved-ci-logs-management-experience-for-multi-line-commands.yml","Inside The Improved Ci Logs Management Experience For Multi Line Commands","en-us/blog/inside-the-improved-ci-logs-management-experience-for-multi-line-commands.yml","en-us/blog/inside-the-improved-ci-logs-management-experience-for-multi-line-commands",{"_path":3019,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3020,"content":3026,"config":3031,"_id":3033,"_type":13,"title":3034,"_source":15,"_file":3035,"_stem":3036,"_extension":18},"/en-us/blog/integrate-external-security-scanners-into-your-devsecops-workflow",{"title":3021,"description":3022,"ogTitle":3021,"ogDescription":3022,"noIndex":6,"ogImage":3023,"ogUrl":3024,"ogSiteName":685,"ogType":686,"canonicalUrls":3024,"schema":3025},"Integrate external security scanners into your DevSecOps workflow","Learn how to bring Snyk scan results into the merge request widget by parsing JSON artifacts and leveraging the SARIF file format.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098768/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%282%29_1khno1AUtxuL6zzmEmjK7v_1750098768560.png","https://about.gitlab.com/blog/integrate-external-security-scanners-into-your-devsecops-workflow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Integrate external security scanners into your DevSecOps workflow\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sam Morris\"}],\n        \"datePublished\": \"2024-04-08\",\n      }",{"title":3021,"description":3022,"authors":3027,"heroImage":3023,"date":3028,"body":3029,"category":830,"tags":3030},[846],"2024-04-08","Each day you build software there is another opportunity for security\nvulnerabilities to creep into production. So it is becoming more important\nthan ever to shift security left and put security tests and the\nvulnerabilities they detect at the forefront of your software development\nlifecycle.\n\n\nWhile GitLab offers a wide range of different security scanners, our\nAI-powered DevSecOps platform provides full visibility into the security of\nyour software. We seek to allow you to not only run scans, but also to view\nresults, bake in approval processes via merge request policies, and display\ncurrent vulnerabilities in your default branch for future triage in our\nVulnerability Report.\n\n\n## How do security scans run?\n\n\nGitLab Ultimate displays your vulnerabilities directly in the merge request\nwidget and it updates on every commit. These scans typically run via jobs in\na pipeline, whether in the project’s `.gitlab-ci.yml` pipeline or in a\nseparately-controlled [compliance\npipeline](https://docs.gitlab.com/ee/user/group/compliance_pipelines.html),\n[security\npolicy](https://docs.gitlab.com/ee/user/application_security/policies/scan-execution-policies.html),\nor [included pipeline\nconfiguration](https://docs.gitlab.com/ee/ci/yaml/includes.html) from a\nseparate .yml file. You can run GitLab’s native security scanners or you can\nrun an external scanner. For this blog post, I took running Snyk scans for a\nspin to see how I could feed the dependency scan results as vulnerability\nrecords back into GitLab. Additionally, I utilized a Static Analysis Results\nInterchange Format (SARIF) converter to read SAST results directly from Snyk\nwithout custom scripting. \n\n\n## Using external scanners\n\n\nGitLab is highly extensible, and the platform allows for you to integrate\nmyriad tools. You can use one of our built-in security scanners, or use an\nexternal scanner via a job in a pipeline or policy. GitLab serves as a\nsingle platform for governance and enforcement, allowing you to bring your\nown scanners and see the results early in the DevSecOps lifecycle.\n\n\nAll you have to do to get started is run a security job, and from there you\ncan obtain the results in the merge request and the vulnerability report.\n\n\n## Run an external scan from GitLab CI\n\n\nIn this example pipeline, I run a Snyk scan externally in the test stage in\na job I overrode called `gemnasium-maven-dependency_scanning`. First, I\ninstall the required packages (npm, Maven, Python3, and Snyk) and then I\nauthorize with my SNYK_TOKEN variable saved in the variables section of my\nproject. Finally, I run a `snyk test` command with the Snyk CLI and output\nthe results to the JSON. This saves my results to the snyk_data_file.json,\nwhich I will parse in a script detailed in the next section and save to the\nrequired artifact file `gl-dependency-scanning-report.json`.\n\n\n```\n\nstages:\n  - test\n\nvariables:\n\n\ninclude:\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml  \n\ngemnasium-maven-dependency_scanning:\n  image: node:latest\n  stage: test\n  services:\n  - openjdk:11-jre-slim-buster\n  before_script:\n    - apt-get update\n    - apt-get install default-jdk -y\n  script:\n    # Install npm, snyk, and maven\n    - npm install -g npm@latest\n    - npm install -g snyk\n    - npm install maven\n    - npm install python3\n    # Run snyk auth, snyk monitor, snyk test to break build and out report\n    - snyk auth $SNYK_TOKEN\n    - chmod +x mvnw\n    - snyk test --all-projects --json-file-output=snyk_data_file.json || true\n    - python3 convert-snyk-to-gitlab.py\n\n  # Save report to artifacts\n  artifacts:\n    when: always\n    paths: \n      - gl-dependency-scanning-report.json\n\n```\n\n\n### Parse the JSON\n\n\nYou can see scan results in the merge request widget from any external\nscanner as long as the artifact of the successful security job is named\nappropriately, for example, `gl-dependency-scanning-report.json`.  \n\n\nHere is an example script that converts the Snyk JSON output to the GitLab\nJSON output. In this example, I open the Snyk data file and load the\nvulnerability data. I create a new list of dependency files and a new list\nof vulnerabilities that contain data GitLab needs to display in the\nvulnerability records, such as the identifier, severity, category,\ndescription, and location. I added a few placeholder sections for required\nfields that I did not need to display in my record. Finally, I saved the\ncontents I parsed out to a new JSON file called\n`gl-dependency-scanning-report.json`, which is the required name for the\nfile to be read by GitLab and have its contents displayed in the widget.\n\n\n```\n\nimport json\n\nfrom types import SimpleNamespace\n\n\nwith open(\"snyk_data_file.json\") as snyk_data_file:\n    snyk_data = json.load(snyk_data_file, object_hook=lambda d: SimpleNamespace(**d))\n\ngitlab_vulns = []\n\ndependency_files = []\n\nfor i in snyk_data:\n    dependency_files.append({\"path\": i.path, \"package_manager\": i.packageManager, \"dependencies\": []})\n    for v in i.vulnerabilities:\n        gitlab_identifiers = []\n        for vuln_type, vuln_names in v.identifiers.__dict__.items():\n            if vuln_names: \n                for vuln_name in vuln_names:\n                    gitlab_identifiers.append({\"type\": vuln_type, \"name\": vuln_name, \"value\": vuln_name.partition(\"-\")[2]})\n                gitlab_vulns.append({\"id\": v.id, \"category\": \"dependency_scanning\", \"severity\": v.severity.capitalize(), \"identifiers\": gitlab_identifiers, \"description\": v.description, \"location\": {\"file\": i.displayTargetFile, \"dependency\": {\"package\": {\"name\": \"PLACEHOLDER\"}, \"version\": \"PLACEHOLDER\"}}})\n\n# Dummy data for scan and dependency files \n\nfull_json = {\"version\": \"15.0.6\", \"dependency_files\": dependency_files,\n\"scan\": {\"analyzer\": {\"id\": \"snyk\", \"name\": \"Snyk\", \"vendor\": {\"name\":\n\"Snyk\"}, \"version\": \"1.0.2\"}, \"scanner\": {\"id\": \"my-snyk-scanner\", \"name\":\n\"My Snyk Scanner\", \"version\": \"1.0.2\", \"vendor\": {\"name\": \"Snyk\"}},\n\"end_time\": \"2022-01-28T03:26:02\", \"start_time\": \"2020-01-28T03:26:02\",\n\"status\": \"success\", \"type\": \"dependency_scanning\"}, \"vulnerabilities\":\ngitlab_vulns}\n\n\nwith open(\"gl-dependency-scanning-report.json\", \"w\") as gitlab_file:\n    json.dump(full_json, gitlab_file, default=vars)\n\n```\n\n\nNow, the vulnerability findings are visible in the merge request widget.\n\n\n![security scanning\ndetection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098776/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098776479.png)\n\n\n## What are SARIF and the SARIF converter?\n\n\nSARIF is a file format for the output of static analysis tools. It is\nincredibly useful when leveraging different security scanners, as all of\ntheir output is formatted the same way. This allows for a generic,\nrepeatable, and scalable approach to application security.\n\n\nThere is a community-maintained [SARIF\nconverter](https://gitlab.com/ignis-build/sarif-converter), which takes\nSARIF files and converts them into ingestible reports. It supports many\nscanners, including Snyk. This converter works for both SAST and code\nquality findings. We are going to focus on SAST for this blog.\n\n\n### Use a SARIF converter to get SAST results\n\n\nTo leverage the SARIF results, first I trigger a Snyk scan as we did in the\nprevious example, but I save the output to a SARIF file. After this, I use\nthe aforementioned converter to create a new JSON file that I save as a\nreport.\n\n\n```\n\nsnyk:\n  image: node:latest\n  stage: test\n  services:\n  - openjdk:11-jre-slim-buster\n  before_script:\n    - apt-get update\n    - apt-get install default-jdk -y\n    - wget -O sarif-converter https://gitlab.com/ignis-build/sarif-converter/-/releases/permalink/latest/downloads/bin/sarif-converter-linux\n    - chmod +x sarif-converter\n  script:\n    # Install npm, snyk, and maven\n    - npm install -g npm@latest\n    - npm install -g snyk\n    - npm install maven\n    # Run snyk auth, snyk monitor, snyk test to break build and out report\n    - snyk auth $SNYK_TOKEN\n    - chmod +x mvnw\n    - snyk test --all-projects --sarif-file-output=snyk.sarif  || true\n    - ./sarif-converter --type sast snyk.sarif snyk.json\n\n  artifacts:\n    reports:\n      sast: snyk.json\n\n```\n\n\nAfter saving the JSON as an artifact, the results are visible in the merge\nrequest widget.\n\n\n![security scanning - image\n2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098776/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098776479.png)\n\n\n## Get started\n\n\nIn this blog post, you learned how to use both custom scripting and a SARIF\nconverter to view external scanner vulnerabilities in the GitLab merge\nrequest widget. These operations can be completed from the pipeline as\nshown, but also from compliance pipelines and pipeline execution policies,\nwhich allow for the enforcement of external scanners. With GitLab Ultimate.\nyou have access to a full DevSecOps platform that allows you to use our\nscanners or bring your own, but build a shift-left workflow that empowers\ndevelopers to remediate vulnerabilities before they hit production.\n\n\n> [Trial GitLab Ultimate today](https://gitlab.com/-/trials/) to begin\nmerging external scanners.\n\n\n## More security scanning resources\n\n\n* [Security scanner integration\ndocumentation](https://docs.gitlab.com/ee/development/integrations/secure.html)\n\n* [How to integrate custom security scanners into\nGitLab](https://about.gitlab.com/blog/how-to-integrate-custom-security-scanners-into-gitlab/)\n\n* [GitLab Trust Center](https://about.gitlab.com/security/)\n",[830,9,1057],{"slug":3032,"featured":6,"template":700},"integrate-external-security-scanners-into-your-devsecops-workflow","content:en-us:blog:integrate-external-security-scanners-into-your-devsecops-workflow.yml","Integrate External Security Scanners Into Your Devsecops Workflow","en-us/blog/integrate-external-security-scanners-into-your-devsecops-workflow.yml","en-us/blog/integrate-external-security-scanners-into-your-devsecops-workflow",{"_path":3038,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3039,"content":3045,"config":3050,"_id":3052,"_type":13,"title":3053,"_source":15,"_file":3054,"_stem":3055,"_extension":18},"/en-us/blog/integrating-azure-devops-scm-and-gitlab",{"title":3040,"description":3041,"ogTitle":3040,"ogDescription":3041,"noIndex":6,"ogImage":3042,"ogUrl":3043,"ogSiteName":685,"ogType":686,"canonicalUrls":3043,"schema":3044},"How to integrate Azure DevOps repositories with GitLab","How to keep your code in an Azure DevOps repository and run CI/CD with GitLab pipelines.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664363/Blog/Hero%20Images/aleksey-kuprikov.jpg","https://about.gitlab.com/blog/integrating-azure-devops-scm-and-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to integrate Azure DevOps repositories with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2020-07-09\",\n      }",{"title":3040,"description":3041,"authors":3046,"heroImage":3042,"date":3047,"body":3048,"category":785,"tags":3049},[1096],"2020-07-09","\n\nRecently we’ve been asked by several people if it is possible to integrate between Azure DevOps/VSTS (Visual Studio Team Services) source code management and GitLab. They are looking for a modern [CI/CD solution](/topics/ci-cd/) like GitLab, but as part of a gradual transition they still need to keep managing their code in Azure DevOps/VSTS. \n\n## Does Azure DevOps integrate with GitLab?\n\nYes, Azure DevOps Services does integrate with GitLab.\n\nAlthough we of course recommend using GitLab CI/CD together with our built-in GitLab SCM, this integration of Azure DevOps source code management and GitLab makes it possible to migrate slowly from Azure DevOps by leaving your code in the Azure [DevOps](/topics/devops/) repository while you adopt GitLab CI/CD. This integration is possible with both the self-managed and SaaS versions of GitLab. The integration works only with Azure DevOps/VSTS git version control. TFVC (Team Foundation Version Control) isn’t supported. \n\n### In GitLab, there are two features that enable this integration:  \n\n[GitLab CI/CD for external repositories](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/) \n\n[Remote repository mirroring](https://docs.gitlab.com/ee/user/project/repository/repository_mirroring.html)\n\nWhat is a repository in DevOps?\n\nCode repositories in tools like GitLab and Azure exist to house all source code. Sometimes these repositories are referenced as a DevOps “repo” or a source repository. Whatever the title, code repositories provide a place where developers can work to ensure high code quality. \nGitLab uses a [git-based repository](/solutions/source-code-management/) for source code management with version control. It lets GitLab users perform code reviews and easily solve developer issues.\n\n## What is the difference between GitLab and Azure DevOps?\n\nAzure DevOps has a range of services for managing the development lifecycle. Some of its main features include agile planning boards, private git repos for source code management, and Azure pipelines.\n\nGitLab is a single platform for the entire DevSecOps lifecycle and includes the following:\n\n- Planning and collaboration\n- Source code management\n- Code reviews\n- CI/CD pipelines\n- Constant security scanning and monitoring\n- Advanced deployments\n- Vulnerability management\n\nGitLab can help manage the entire DevSecOps lifecycle to deliver software quickly and efficiently while bolstering security and compliance.\n\n## How do I connect to Azure from GitLab?\n\nIt may take some time to fully move over from Azure to GitLab for source code management. To smooth the transition, there are simple steps to connect to the Azure integration from GitLab.\n\n1. Create a new project in GitLab by clicking the New Project button  ![Create new project ](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado1.png){: .large.center}\n\n2. Choose the ‘CI/CD for external repo’ tab, and click on Repo by URL.  ![CI/CD for external repo](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado2.png){: .large.center}\n\n3. Open your repository in Azure DevOps and click Clone  ![Getting clone url ](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado3.png){: .large.center}\n\n4. Copy the URL. If your repository is private, you will need to generate Git credentials – just click this button and copy the username and password.  ![Credentials](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado4.png){: .large.center}\n\n5. Paste the URL in GitLab under the Git repository URL, give it a name, set the visibility level, and click create project. Add the username and password in case your Azure DevOps repository is private. Note: The repository must be accessible over http://, https:// or git://. When using the http:// or https:// protocols, please provide the exact URL to the repository. HTTP redirects will not be followed.  ![Create project form](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado5.png){: .large.center}\n\n6. Your project is now successfully Mirrored to GitLab. Now branches, tags, and commits will be synced automatically to GitLab. \n\n7. To configure a CI/CD pipeline there are two options:\n\nBefore pushing your first commit, open the CI/CD settings in GitLab and enable Auto DevOps.  It will set the CI/CD configuration, so each commit in Azure Repos will trigger a CI/CD  pipeline in GitLab which will build, test, and deploy your app.  ![Auto DevOps settings](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado6.png){: .shadow.large.center}\n \nAlternatively, in case you want to define the pipeline configuration yourself instead of using the Auto DevOps, add [.gitlab-ci.yml](https://docs.gitlab.com/ee/ci/yaml/) file to  your repository root directory. The Yaml code should include your [CI/CD definitions](/blog/guide-to-ci-cd-pipelines/). Once this file is included in the root directory a CI/CD pipeline will be triggered for each commit. If you are not familiar with .gitlab-ci.yml, start by creating a file with the name .gitlab-ci.yml and paste the below code to it. This code includes build and test stages, and a job that displays text to the console in each stage. Later on you can add additional scripts to each job, and also add additional jobs and stages. To create more complex pipelines, you can [use the pipeline templates](https://docs.gitlab.com/ee/ci/yaml/#includetemplate) that are [shipped with GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates) instead of starting it from scracth.\n\n```\nstages:\n  - build\n  - test \n  \nbuild:\n  stage: build\n  script:\n    - echo \"Build job\"\n\ntest:\n  stage: test\n  script:\n    - echo \"Test job\"\n```\n\nThat’s it, you are all set! \n\n## Suggested development flow \n\n![Development flow diagram](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado_7_2.png){: .shadow.large.center}\n\n1. CODE (Developer IDE of choice) Developer uses the favorite IDE to develop code, clones the repo to the workstation and creates a branch.  ![Visual Studio Code](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado8.png){: .shadow.large.center}\n\n2. COMMIT (GIT) After the feature is developed/the bug is fixed, the developer pushes the work to the Azure Repository server.  ![Azure DevOps Repos](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado9.png){: .shadow.large.center}\n\n3. BUILD (GitLab) The branch with the commit history will be mirrored to GitLab. The CI/CD pipeline will be triggered. The pipeline will build the code.  ![GitLab pipeline graph](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado10.png){: .shadow.large.center}\n\n    Artifacts will be created, and be available for download.  ![Artifacts](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado11.png){: .shadow.large.center}\n\n    If Auto DevOps is enabled, a container image will be created and be pushed to the built-in Container Registry.  ![GitLab Container Registry](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado12.png){: .shadow.large.center}\n\n    In case a package registry is enabled in the project, packages will be published to the designated package manager.  ![GitLab Package Registry](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado13.png){: .shadow.large.center}\n\n4. TEST (GitLab) Security scans, license scans, and other tests are executed as part of the CI pipeline.  ![GitLab scans](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado14.png){: .shadow.large.center}\n\n5. REVIEW & PULL REQUEST (GitLab & Azure DevOps repos) Review pipeline results in GitLab and if the pipeline passed without errors, and the new change hasn’t introduced new vulnerabilities, the developer creates a pull request in Azure DevOps. A code review is started and the developer might need to make a few changes before merging to master. Each commit will trigger a CI/CD pipeline in GitLab.  ![Azure DevOps pull request](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado15.png){: .shadow.large.center}\n\n6. MERGE (Azure DevOps Repos and GitLab) The Azure DevOps pull request is approved and the branch will be merged to the master branch in the Azure DevOps Repository.\n\nDepending on your pipeline configuration, this merge to the master branch will trigger the CI/CD pipeline in GitLab to validate the merge results, build new packages and container images, and then deploy them.  ![GitLab CI/CD pipeline graph](https://about.gitlab.com/images/blogimages/ado_and_gitlab/ado16.png){: .shadow.large.center}\n\n## Development workflow demonstration \n\n\u003Ciframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/HfpP2pEmkoM\" frameborder=\"0\" allow=\"accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen>\u003C/iframe>\n\n## A solution worth trying \n\nGitLab offers a leading source code management and CI/CD solution in one application which many [GitLab customers](/customers/) use together because of the power of this combination. However, we know that sometimes there are constraints that do not allow teams to migrate their repository to GitLab SCM, at least not right away. For these situations, even if it is only temporary, we offer the capability of GitLab CI/CD for external repositories illustrated here. \n\n\n**Read more about GitLab CI/CD:**\n\n[Forrester report compares between leading CI/CD tools](https://about.gitlab.com/analysts/forrester-cloudci19/)\n\n[Autoscale GitLab CI with AWS Fargate](/blog/introducing-autoscaling-gitlab-runners-on-aws-fargate/)\n\n[Case Study - how Goldman Sachs improved from 1 build every two weeks to over a thousand per day](https://about.gitlab.com/customers/goldman-sachs/)\n\nCover image by [Aleksey Kuprikov](https://unsplash.com/@alekskuprfilmz) on [Unsplash](https://unsplash.com/)\n{: .note}\n\n\n",[108,996,9],{"slug":3051,"featured":6,"template":700},"integrating-azure-devops-scm-and-gitlab","content:en-us:blog:integrating-azure-devops-scm-and-gitlab.yml","Integrating Azure Devops Scm And Gitlab","en-us/blog/integrating-azure-devops-scm-and-gitlab.yml","en-us/blog/integrating-azure-devops-scm-and-gitlab",{"_path":3057,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3058,"content":3064,"config":3070,"_id":3072,"_type":13,"title":3073,"_source":15,"_file":3074,"_stem":3075,"_extension":18},"/en-us/blog/introducing-ci-components",{"title":3059,"description":3060,"ogTitle":3059,"ogDescription":3060,"noIndex":6,"ogImage":3061,"ogUrl":3062,"ogSiteName":685,"ogType":686,"canonicalUrls":3062,"schema":3063},"Introducing CI/CD components and how to use them in GitLab","Learn the main benefits for using CI/CD components in your CI/CD pipelines and how to achieve them.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667676/Blog/Hero%20Images/buildingblocks.jpg","https://about.gitlab.com/blog/introducing-ci-components","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Introducing CI/CD components and how to use them in GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Dov Hershkovitch\"}],\n        \"datePublished\": \"2023-07-10\",\n      }",{"title":3059,"description":3060,"authors":3065,"heroImage":3061,"date":3067,"body":3068,"category":785,"tags":3069},[3066],"Dov Hershkovitch","2023-07-10","Welcome to the third blog in our series on GitLab's CI/CD components! If you\nhaven't already, we encourage you to read \"[How to build reusable CI/CD\ntemplates](https://about.gitlab.com/blog/how-to-build-reusable-ci-templates/)\"\nand \"[Use inputs in includable\nfiles](https://about.gitlab.com/blog/use-inputs-in-includable-files/)\"\nto gain a comprehensive understanding of these exciting new capabilities. In\nthis blog post, we'll dive in and explore the power of GitLab's CI/CD\ncomponents in revolutionizing CI/CD workflows. We'll also provide a glimpse\ninto the future of GitLab's CI/CD ecosystem, including the upcoming release\nof the [CI/CD\ncatalog](https://docs.gitlab.com/ee/architecture/blueprints/ci_pipeline_components/),\na framework containing a collection of these components. With these moves,\nGitLab is taking a significant step towards streamlining pipeline\nconfigurations and enhancing reusability.\n\n\n### CI/CD components\n\nIn [GitLab\n16.1](https://about.gitlab.com/releases/2023/06/22/gitlab-16-1-released/),\nan exciting experimental feature called CI/CD components was introduced.\nCI/CD components are reusable, single-purpose building blocks that abstract\naway pipeline configuration units.\n\n\nBy leveraging the power of CI/CD components, users can unlock several key\nbenefits:\n\n1. **Reusability and abstraction.** CI/CD components allow pipelines to be\nassembled using abstractions instead of defining all the details in one\nplace. With components encapsulating implementation details, developers can\nfocus on composing pipelines using pre-built, reusable blocks. This approach\npromotes modularity, code reusability, and simplifies pipeline maintenance.\n\n2. **Flexibility with input.** Components support input parameters, enabling\ncustomization based on pipeline contexts, making them adaptable and reusable\nacross various pipeline stages. Developers gain the ability to build a\ndynamic CI/CD catalog that is tagged and versioned, providing better control\nand compatibility. Developers can reference specific component versions,\nensuring stability and reproducibility. By leveraging version tags, teams\ncan maintain consistency in their pipelines while easily upgrading to newer\nversions when desired.\n\n4. **High-quality standards through testing.** Testing components as part of\nthe development workflow to ensure quality maintains high standards is\nstrongly recommended. By incorporating testing into the CI/CD process,\ndevelopers can verify the reliability and functionality of components,\nidentify and fix issues early on, and deliver more robust and dependable\npipelines.\n\n5. **The CI/CD catalog.** A centralized repository of components, the CI/CD\ncatalog is set to be released soon, and will act as a treasure trove of\ncomponents curated to cover a wide range of use cases. This centralized\nrepository offers developers a one-stop shop for discovering, integrating,\nand sharing components. Teams can benefit from a growing catalog of\npre-built, quality-tested components, saving time and effort in configuring\ntheir pipelines.\n\n\nIn the previous blog posts, we discussed the main benefits for the first two\npoints (which are also available with CI/CD templates), but now let's dig\ndeeper into components and how they could revolutionize the way you\nconstruct your pipelines.\n\n\n### Testing a CI/CD component\n\nAs software development continues to evolve, ensuring the reliability and\nquality of code components becomes increasingly vital.\n\n\nOne of the main benefits of using components is the ability to thoroughly\ntest components before software is officially released, enabling a more\nrobust and streamlined development process. In our context, a released\ncomponent is versioned and will follow a structured syntax, allowing for\nseamless integration within pipelines. \n\n\n```yaml\n\ninclude:\n  - components: /path/to/project@\u003Cversion> \n```\n\nOne of the unique benefits of our CI/CD components is the flexibility they\noffer. DevSecOps teams can opt in for an \"unofficial\" release by appending\n`@commit_SHA`, allowing them to experiment and iterate on their code before\nmaking it an official release.\n\n\n```yaml\n\ninclude:\n  - components: /path/to/project@\u003Ccommit_SHA> \n```\n\nTo make a component an official release, users must tag it, essentially\ncreating a versioned snapshot. The tagged release will then be made\navailable in our comprehensive CI/CD catalog (launching soon), providing\nusers with easy access to a range of thoroughly tested and approved\ncomponents. To ensure the stability and reliability of your CI/CD\ncomponents, it is crucial to thoroughly test them. DevSecOps teams can\nleverage the power of our pipeline by utilizing the commit_SHA identifier to\nrun comprehensive tests. If the pipeline successfully passes all tests, they\ncan proceed to tag the component, signifying its readiness for release.\n\n\nBy configuring a release job based on the tagged version, DevSecOps teams\ncan confidently incorporate the official component into their projects,\nknowing that it has undergone testing and validation. To learn more about\nhow to test components, you can check out our\n[documentation](https://docs.gitlab.com/ee/ci/components/#test-a-component)\nor watch this walkthrough video:\n\n\n\u003Cfigure class=\"video_container\">\n\n\u003Ciframe width=\"1870\" height=\"937\"\nsrc=\"https://www.youtube.com/embed/Vw8-ce8LNBs\" title=\"\" frameborder=\"0\"\nallow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope;\npicture-in-picture\" allowfullscreen>\u003C/iframe>\n\n\u003C/figure>\n\n\n### Versioning and tagging\n\nAs mentioned in the previous section, DevSecOps teams can leverage the\n`@version` or the `@commit_SHA` to refer to a component in their pipeline.\nAnother option to refer to a component is by leveraging the `@latest`.\n\n\n```yaml\n\ninclude:\n  - components: /path/to/project@latest\n```\n\nThis will use the latest official (tagged) available components. When used\nin a pipeline in combination with reliable tests, you can guarantee that\nyour components used in a pipeline will always be tested and verified.\n\n\n### On the horizon: CI/CD catalog\n\nOne of the biggest benefits of using components is yet to be seen and will\nbe available with the launch of our CI/CD catalog. The catalog will allow\nusers to search, find, and understand how to use components that are\navailable across their organization, setting a framework for them to\ncollaborate on pipeline constructs so that they can be evolved and improved\nover time. Stay tuned!\n\n\n### Dogfooding components \n\nAt GitLab, we believe in [dogfooding our own\nproduct](https://handbook.gitlab.com/handbook/values/#dogfooding). To\ndemonstrate the power and practicality of CI/CD components, we have\nconverted some of our GitLab templates into components and asked our\ninternal team to use them and provide additional feedback. By doing so, we\nare actively using and testing components in real-world scenarios,\nuncovering insights, and continuously improving their functionality. In this\n[group](https://gitlab.com/gitlab-components), we’ve converted Code Quality,\nContainer Scanning and SAST templates into CI/CD components and asked\ninternal teams to use them.\n\n\nThrough this dogfooding process, we are not only validating the\neffectiveness of CI/CD components but also gaining invaluable experience and\nfeedback to refine and enhance our offering. It's a testament to our\ncommitment to providing practical and reliable solutions for our users. You\ncan view the ongoing discussions between the internal teams in this\n[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390656).\n\n\n### Call for action\n\nThe CI/CD component catalog is currently in an experimental phase, so we\nadvise against using it in a production environment at this time. There is a\nhigh probability of changes being made to it. We are currently working on\nreorganizing the folder structure of the components to prepare for the\nlaunch of the CI/CD catalog. You can stay updated on our progress by\nfollowing our [epic](https://gitlab.com/groups/gitlab-org/-/epics/10728), or\nlet us know what you think in this dedicated [feedback\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/407556).\n\n\n### What's next\n\nGitLab's CI/CD component catalog and its accompanying CI/CD components\nfeature are ushering in a new era of streamlined pipeline configurations. By\nembracing reusability, abstraction, input flexibility, versioning, and a\ncentralized catalog, developers can build efficient, adaptable, and\nmaintainable CI/CD workflows. The CI/CD component catalog empowers teams to\naccelerate their software delivery, collaborate effectively, and leverage\nthe full potential of GitLab's CI/CD capabilities.\n\n\nStay tuned for the launch of the CI/CD catalog, where you'll gain access to\nan extensive collection of components, unlocking new possibilities for your\npipelines. GitLab remains committed to empowering developers with\ncutting-edge tools, driving innovation, and simplifying the complexities of\nmodern software development.\n\n\n> Learn more about the CI/CD Catalog and components:\n\n>  \n\n> - [CI/CD Catalog goes GA: No more building pipelines from\nscratch](https://about.gitlab.com/blog/ci-cd-catalog-goes-ga-no-more-building-pipelines-from-scratch/)\n\n> \n\n> - [A CI/CD component builder's\njourney](https://about.gitlab.com/blog/a-ci-component-builders-journey/)\n\n>\n\n> - [FAQ: GitLab CI/CD\nCatalog](https://about.gitlab.com/blog/faq-gitlab-ci-cd-catalog/)\n\n>\n\n> - [Documentation: CI/CD components and CI/CD\nCatalog](https://docs.gitlab.com/ee/ci/components/)\n\n> \n\n\nCover image by [Alexander\nGrey](https://www.pexels.com/photo/assorted-color-bricks-1148496/) on\n[Pexels](https://www.pexels.com).\n\n{: .note}\n",[9,108,696],{"slug":3071,"featured":6,"template":700},"introducing-ci-components","content:en-us:blog:introducing-ci-components.yml","Introducing Ci Components","en-us/blog/introducing-ci-components.yml","en-us/blog/introducing-ci-components",{"_path":3077,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3078,"content":3084,"config":3089,"_id":3091,"_type":13,"title":3092,"_source":15,"_file":3093,"_stem":3094,"_extension":18},"/en-us/blog/introducing-the-gitlab-ci-cd-catalog-beta",{"title":3079,"description":3080,"ogTitle":3079,"ogDescription":3080,"noIndex":6,"ogImage":3081,"ogUrl":3082,"ogSiteName":685,"ogType":686,"canonicalUrls":3082,"schema":3083},"Introducing the GitLab CI/CD Catalog Beta","Discover, reuse, and contribute CI/CD components effortlessly, enhancing collaboration and efficiency when creating pipeline configurations.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099399/Blog/Hero%20Images/Blog/Hero%20Images/security-pipelines_4UHVIJlePT8rEzjvYkGYvi_1750099398604.jpg","https://about.gitlab.com/blog/introducing-the-gitlab-ci-cd-catalog-beta","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Introducing the GitLab CI/CD Catalog Beta\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2023-12-21\",\n      }",{"title":3079,"description":3080,"authors":3085,"heroImage":3081,"date":3086,"body":3087,"category":785,"tags":3088},[1096],"2023-12-21","DevSecOps is all about speed – achieving rapid progress in software development. To succeed in DevSecOps, organizations require a well-functioning CI/CD pipeline that teams can utilize to automate their development workflows.\n\nHowever, crafting pipeline configurations with YAML can be intricate and challenging because YAML isn't a programming language, Developers may find themselves reinventing the wheel each time they try to create new configurations because they don't have visibility into existing configurations and work that others may have already done, resulting in inefficiency.\n\n[GitLab 16.7](https://about.gitlab.com/releases/2023/12/21/gitlab-16-7-released/) introduces the [CI/CD Catalog](https://docs.gitlab.com/ee/ci/components/#cicd-catalog) (Beta), with the goal of enhancing developer efficiency by addressing three main questions developers encounter when creating pipeline configurations:\n\n* Discoverability: Has someone already created a configuration for my task, and where can I find it?\n* Reusability: Once I find a suitable pipeline, how do I use it effectively?\n* Ease of contribution: I've created a useful configuration; how can I easily share it with the GitLab community?\n\n## What is the GitLab CI/CD Catalog?\n\nThe CI/CD Catalog serves as a centralized hub for developers and organizations to share pre-existing [CI/CD components](https://docs.gitlab.com/ee/ci/components/) and to discover reusable configurations that others may have already developed. Every component published by users will be part of a public catalog accessible to all users, regardless of their organization or project. \n\nThis approach promotes cross-organization collaboration, allowing the entire GitLab community to benefit from the wealth of CI components available. It's a powerful step forward in sharing knowledge among GitLab users, enabling developers to harness the collective expertise of the platform.\n\n## Easy component creation and publishing\n\nIn addition to reusing components, developers can contribute to the GitLab CI/CD community by creating their own components and publishing them in the catalog. This ensures that others can benefit from their expertise and encourages collaboration across the platform.\n\n## How to discover and use components\n\n**1. Opening the CI/CD Component Catalog**\n\nClick on “Search or go to...”\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099406962.png)\n\nOpen the catalog by navigating to “Explore > CI/CD Catalog” or visit this [catalog page](https://gitlab.com/explore/catalog).\n\nUpon accessing the catalog, you'll find a list of CI/CD components projects contributed by your team, organization, or the wider GitLab community.\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099406963.png)\n\n**2. Browsing components**\n\nNavigate through the list of components in the CI/CD Catalog or use the Search bar to find components related to a specific topic.\n\nEach component project contains one or multiple components. Opening a component project will display its documentation, providing details on all available components. This includes insights into how to use each component and understanding the expected input parameters.\n\n**3. Include the selected components in your .gitlab-ci.yml**\n\nNow that you've explored the catalog and selected the desired CI/CD components, integrate them into your project's CI/CD pipeline.\n\nFollow these steps to update your .gitlab-ci.yml file:\n\n1. Open the .gitlab-ci.yml file in your project for editing.\n2. Use the include keyword to add the selected components to your CI configuration. \n3. Ensure that the paths to the component YAML files are correct and specify the appropriate version for each component.\n4. In case the components have input parameters, review the component’s documentation to understand which inputs are required, and add them to your CI configuration.\n5. Save and commit your changes to the .gitlab-ci.yml file.\n\nHere is an example of YAML code that demonstrates how to include a few components and use them with input parameters.\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_1.15.48_PM_aHR0cHM6_1750099406965.png)\n\n## How to create and publish components\n\nHave you crafted a valuable configuration that you'd like to share and contribute to your team or the GitLab community? Here are the six steps to make it happen:\n\n**Step 1: Create a new project and set it as a component project**\n\n1. On the left sidebar, select **Search or go to** and find your project.\n2. On the left sidebar, select Settings > General.\n3. Expand Visibility, project features, permissions.\n4. Scroll down to CI/CD Catalog resource and select the toggle to set the project as a CI/CD Catalog resource.\n5. Ensure that your project description is filled out; this information will be showcased in the catalog, providing users with insights into the purpose and functionality of your components.\n6. Create a .gitlab-ci.yml file in the root of the repository. You will need this file to test and release the components as described in steps 4 and 5 below. Note: This step only needs to be done once for any project that contains components.\n\n**Step 2: Create the components**\n\n1. Create a /templates folder in the root directory of the project.\n2. In this templates directory, create one YAML template file (ending in .yml) for each component.   \n3. The template can optionally include a description of input arguments using the `spec` keyword if the component requires input parameters, and the definition of jobs, that may include references to values using the interpolation format $[[ inputs.input-name ]]. Ensure you use three dash lines between the spec header, and job definitions.\n\nHere is an example of a `deploy.yml` template that gets input parameters:\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_11.34.20_AM_aHR0cHM6_1750099406966.png)\n\nIn this template, we've defined two input parameters, `stage` and `environment`, both with default values. In the content section, a job is defined that interpolates these input arguments.\n\n**Step 3: Create components documentation** \n\nCreate a README.md file in the root of the project, including information about the components. Explain the component's functionality, detail input parameters, and provide illustrative examples. This ensures clarity for component consumers on how to use them.\n\nThis is an example of component documentation:\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099406967.png)\n\nAdditional information can be found in our [CI/CD components](https://docs.gitlab.com/ee/ci/components/index.html#components-repository) documentation. \n\n**Step 4: Add tests to the components (recommended)**\n\nDeveloping a component follows a standard software development cycle with stages like build, test, and deploy. It's highly recommended to test your components before publishing them. Check out this example test, which queries the GitLab REST API to check whether a component job has been added to the pipeline. Feel free to use it, and consider adding more tests to ensure your components work as expected.\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_12.32.53_PM_aHR0cHM6_1750099406968.png)\n\nInclude all your test jobs in the **.gitlab-ci.yml** file in your Catalog project.\n\n**Step 5: Prepare your CI/CD configuration for publishing**\n\n1. Create a release job in the **.gitlab-ci.yml** file in the component project using the `Release` keyword.  See the job example:\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_12.34.27_PM_aHR0cHM6_1750099406969.png)\n\n__Note:__ Do not \"create release\" from GitLab UI since this soon won't be supported for a Component Catalog.\n\n2. We recommend adding this rule in the Release job; this will automatically trigger the Release job only when creating a git tag starts with digits in the project, following semantic release conventions (1.0.0 for example).\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_1.21.30_PM_aHR0cHM6_1750099406970.png)\n\n3. So this is how we recommend your job to look: \n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_12.37.09_PM_aHR0cHM6_1750099406970.png)\n\n4. To manually release components, add manual rule as below, so when the pipeline is triggered, someone will need to manually run the release job. \n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_12.38.18_PM_aHR0cHM6_1750099406971.png)\n\nHere is the release job with the `when:manual` rule:\n\n![component catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099407/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2023-12-19_at_12.41.00_PM_aHR0cHM6_1750099406972.png)\n\n**Step 6: Publish your components**\n\nOnce you are satisfied with your components, and all tests have passed successfully, it's time to publish a new version by creating a git tag, so they will be available in the CI/CD Catalog.\n\n1. Create a Git tag using the semantic versioning format \"MAJOR.MINOR.PATCH\". \n\n2. You can create tags through the UI by navigating to Code -> Tags -> New Tag, or via the CLI using `git tag`. \n\n3. Creating the tag will trigger a pipeline that runs the Release job if all tests pass successfully. The component project will then be assigned the version you defined in the tag, and it will appear in the catalog.\n\n### Example projects\n\n* [GitLab official components](https://gitlab.com/components)\n\n### Documentation \n\nFor more details on using components from the CI/CD Catalog and maximizing their potential within your projects, refer to the official [CI/CD Catalog documentation](https://docs.gitlab.com/ee/ci/components/#cicd-catalog). This documentation provides in-depth insights into the functionality.\n\n> [Take a tour](https://gitlab.navattic.com/cicd-catalog) of the GitLab CI/CD Catalog.\n\n_A special thank you to [Dov Hershkovitch](https://about.gitlab.com/company/team/#dhershkovitch) and [Fabio Pitino](https://gitlab.com/fabiopitino) for their invaluable content reviews and contributions to this blog post._",[108,891,789,9],{"slug":3090,"featured":6,"template":700},"introducing-the-gitlab-ci-cd-catalog-beta","content:en-us:blog:introducing-the-gitlab-ci-cd-catalog-beta.yml","Introducing The Gitlab Ci Cd Catalog Beta","en-us/blog/introducing-the-gitlab-ci-cd-catalog-beta.yml","en-us/blog/introducing-the-gitlab-ci-cd-catalog-beta",{"_path":3096,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3097,"content":3103,"config":3111,"_id":3113,"_type":13,"title":3114,"_source":15,"_file":3115,"_stem":3116,"_extension":18},"/en-us/blog/introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci",{"title":3098,"description":3099,"ogTitle":3098,"ogDescription":3099,"noIndex":6,"ogImage":3100,"ogUrl":3101,"ogSiteName":685,"ogType":686,"canonicalUrls":3101,"schema":3102},"OIDC simplifies GitLab CI/CD authentication with Google Cloud","OpenID Connect can sometimes be complex, but it's the safer and recommended way to authenticate your GitLab pipeline with Google Cloud. This tutorial shows you how.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749669863/Blog/Hero%20Images/security-pipelines.jpg","https://about.gitlab.com/blog/introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How OIDC can simplify authentication of GitLab CI/CD pipelines with Google Cloud\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Hiroki Suezawa\"},{\"@type\":\"Person\",\"name\":\"Dhruv Jain\"}],\n        \"datePublished\": \"2023-06-28\",\n      }",{"title":3104,"description":3099,"authors":3105,"heroImage":3100,"date":3108,"body":3109,"category":830,"tags":3110},"How OIDC can simplify authentication of GitLab CI/CD pipelines with Google Cloud",[3106,3107],"Hiroki Suezawa","Dhruv Jain","2023-06-28","In recent years, the [integration of cloud services and GitLab through\nGitOps](https://about.gitlab.com/blog/the-ultimate-guide-to-gitops-with-gitlab/)\nhas become very common. Applications are now continuously tested and\ndeployed through [continuous integration and delivery\n(CI/CD)](https://about.gitlab.com/topics/ci-cd/); cloud environments are\nmanaged in code through Infrastructure as Code (IaC) using tools like\nTerraform; and GitLab CI is used as a core tool to perform these GitOps\nprocesses.\n\n\nAt the same time, [software supply chain\nattacks](https://about.gitlab.com/blog/the-ultimate-guide-to-software-supply-chain-security/)\nhave increased. To reduce the risk of an attack, the use of OpenID Connect\n([OIDC](https://docs.gitlab.com/ee/integration/openid_connect_provider.html#introduction-to-openid-connect))\nauth is recommended, and GitLab 15.7 introduced [ID\ntokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html),\na mechanism for secure OIDC integration.\n\n\nHowever, OIDC integration can be complex for beginners and can be difficult\nto configure properly. Therefore, GitLab's Infrastructure Security Team has\ncreated a Terraform module for configuring Google Cloud and a CI template\nfor GitLab CI so GitLab CI and Google Cloud can be securely integrated.\n\n\nThis tutorial explains how to use [these OIDC\nmodules](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules).\n\n\n## Why OIDC?\n\nThe integration between Google Cloud and GitLab CI has often been done by\nadding a static key of the service account in Google Cloud to the\nenvironment variables of CI. However, this method has the following\nproblems:\n\n\n- The risk of compromise is high because the same key can be used to\nmanipulate the cloud environment over time.\n\n- Because static keys are portable, there is no link between the key and the\nenvironment in which it is used, making it difficult to identify where the\nkey is being used.\n\n\nOIDC authentication can solve the above problems by providing the following\nbenefits:\n\n- No need to issue static keys, eliminating the need for long-term key\nmanagement.\n  - It also eliminates the compliance need of rotating the secrets every few months.\n- Low risk of leakage due to temporary tokens issued.\n\n- Because the CI used is tied to the Google Cloud environment, it is\npossible to properly manage where the service account is used.\n\nIn addition, other settings such as CI and CD isolation can be configured\nusing [the claims provided by GitLab\nCI](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html).\n\n\n## OIDC authentication with Google Cloud\n\nThe OIDC integration between Google Cloud and GitLab CI works as follows:\n\n\n- Preparation (areas to configure in Terraform in OIDC models)\n  1. Create a service account in Google Cloud for CI integration and set up the appropriate roles.\n  1. Create a Google Cloud Workload Identity pool and provider, and configure integration with GitLab CI.\n  1. Assign the Workload Identity User role to the service account.\n\n\n\n\n![Simplified\ndiagram](https://about.gitlab.com/images/blogimages/2023-06-30-introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci/oidc-auth-diagram.png){:\n.shadow}\n\n\nGitLab CI in action (simplified by the GitLab CI template in OIDC modules)\n\n{: .note .text-center}\n\n\nGoogle Cloud authenticates using an ID token issued on GitLab CI, so there\nis no need to issue a Google Cloud service account key.\n\n\n## How to use a Terraform module\n\nThe process of configuring a Terraform module to establish a connection\nbetween Google Cloud and GitLab using OIDC is fairly simple. This module\ntakes care of the following steps:\n\n1. Create the Google Cloud Workload Identity Pool.\n\n1. Create a Workload Identity Provider.\n\n1. Grant permissions for service account impersonation.\n\n\nNote: Your account must have at least the Workload Identity Pool Admin\npermission on the Google Cloud project.\n\n\n```terraform\n\n# terraform\n\nmodule \"gl_oidc\" {\n source = \"gitlab.com/gitlab-com/gcp-oidc/google\"\n version = \"3.0.0\"\n google_project_id = GOOGLE_PROJECT_ID\n gitlab_project_id = GITLAB_PROJECT_ID\n oidc_service_account = {\n   \"sa\" = {\n     sa_email  = \"SERVICE_ACCOUNT_EMAIL\"\n     attribute = \"attribute.project_id/GITLAB_PROJECT_ID\"\n   }\n }\n}\n\n```\n\n\nThe above sample module can be used to configure OIDC. There are some\nadditional parameters that can be used to configure this module further (a\ndetailed list and description of those parameters can be found\n[here](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/tree/main/#configure-gitlab-for-oidc-integration-using-terraform-module)).\n \nBy default, all branches of the project are authenticated to Google Cloud,\nbut you can specify more granular conditions, such as the branch name of the\ncommit that triggered the CI, or authenticating only with a specific tag.\n\n\nFurther settings can be made by changing the following attribute settings in\naccordance with the ID token claim:\n\n\n```\n  oidc_service_account = {\n    \"sa\" = {\n      sa_email  = \"SERVICE_ACCOUNT_EMAIL\"\n      attribute = \"attribute.project_id/GITLAB_PROJECT_ID\"\n    }\n```\n\n\nCode files for this module are available\n[here](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/tree/main/terraform-modules/gcp-oidc).\n\n\n## How to use the CI template\n\n[The CI\ntemplate](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/blob/main/templates/gcp_auth.yaml)\nmakes GitLab CI very easy for Google Cloud OIDC authentication. This CI\ntemplate supports [Application Default\nCredentials](https://cloud.google.com/docs/authentication/application-default-credentials)\nand can be used from IaC such as Terraform, CLI such as gcloud, and SDKs in\nPython and Go.\n\n\nFor example, if you want to use the CI template for Terraform, you can\nwrite:\n\n\n```\n\n# You should upgrade to the latest version. You can find the latest version\nat\nhttps://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/releases\n\ninclude:\n  - remote: 'https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/raw/3.0.0/templates/gcp_auth.yaml'\n\nterraform:\n  image:\n    name: hashicorp/terraform:1.5.3\n    entrypoint:\n      - /usr/bin/env\n      - \"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"\n  extends: .google-oidc:auth\n  variables:\n    WI_POOL_PROVIDER: //iam.googleapis.com/projects/GOOGLE_PROJECT_ID/locations/global/workloadIdentityPools/WORKLOAD_IDENTITY_POOL/providers/WORKLOAD_IDENTITY_POOL_PROVIDER\n    SERVICE_ACCOUNT: SERVICE_ACCOUNT_EMAIL\n  script:\n    - terraform init\n    - terraform plan\n```\n\n\n### Required variables\n\n- WI_POOL_PROVIDER(under .google-oidc:) - Full canonical resource name of\nthe workload identity pool provider. This value must be written under\n.google-oidc: like this.\n\n- SERVICE_ACCOUNT - Service Account email address\n\n\nA detailed list and description of those parameters can be found\n[here](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/blob/main/README.md#using-oidc-in-pipelines).\n\n\nAs a note, you cannot use `before_script` in the job that uses this template\nbecause the way GitLab CI works will result in OIDC code being overwritten.\nCI template uses `before_script` to perform the initial configuration of\nOIDC.\n\n\nCode samples for this module are available\n[here](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/tree/main/samples/ci/gcp).\n\n\n## Next steps\n\nThis article has introduced OIDC modules for OIDC integration and secure\nauthentication between Google Cloud and GitLab CI. In short, we are doing\nthe following steps:\n\n\n1. Setting up a service account\n\n1. Granting permissions to the service account\n\n1. Running the Terraform module\n\n1. Setting up CI pipeline\n\n\nYou can find the relevant sample for the above steps\n[here](https://gitlab.com/gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules/-/tree/main/samples).\n\n\nAlso, GitLab is currently developing a [CI Catalog and CI\nComponents](https://about.gitlab.com/blog/use-inputs-in-includable-files/).\nWe plan to support them.\n\n\nThe GitLab Infrastructure Security Team will continue to improve the modules\nas we receive feedback, and we hope to consider and release components that\nmaintain a high level of security and usability for both internal and\nexternal use. \n\n\n## Read more\n\n- [Configure OIDC with GCP Workload Identity\nFederation](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/)\n\n- [Workload Identity Federation on Google\nCloud](https://cloud.google.com/iam/docs/workload-identity-federation)\n\n- [Terraform for\ngoogle_iam_workload_identity_pool_provider](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/iam_workload_identity_pool_provider)\n\n- [OIDC Authentication using ID\ntokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html)\n",[9,108,830,1898],{"slug":3112,"featured":6,"template":700},"introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci","content:en-us:blog:introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci.yml","Introduction Of Oidc Modules For Integration Between Google Cloud And Gitlab Ci","en-us/blog/introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci.yml","en-us/blog/introduction-of-oidc-modules-for-integration-between-google-cloud-and-gitlab-ci",{"_path":3118,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3119,"content":3125,"config":3130,"_id":3132,"_type":13,"title":3133,"_source":15,"_file":3134,"_stem":3135,"_extension":18},"/en-us/blog/ios-cicd-with-gitlab",{"title":3120,"description":3121,"ogTitle":3120,"ogDescription":3121,"noIndex":6,"ogImage":3122,"ogUrl":3123,"ogSiteName":685,"ogType":686,"canonicalUrls":3123,"schema":3124},"Tutorial: iOS CI/CD with GitLab","Learn how to create an automated CI/CD pipeline using GitLab and fastlane.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749669340/Blog/Hero%20Images/john-cameron-DgRb7aAGK4k-unsplash.jpg","https://about.gitlab.com/blog/ios-cicd-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: iOS CI/CD with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Darby Frey\"}],\n        \"datePublished\": \"2023-06-07\",\n      }",{"title":3120,"description":3121,"authors":3126,"heroImage":3122,"date":3127,"body":3128,"category":785,"tags":3129},[910],"2023-06-07","Creating an automated [CI/CD](https://docs.gitlab.com/ee/ci/) pipeline for\nan Apple iOS application can be challenging. Configuring build environments\nand managing code signing can be very time-consuming and error-prone, and\nwhen you get that all working, you still need a way to send your app to\nApple.\n\n\nGitLab makes this much easier with [GitLab Mobile\nDevOps](https://docs.gitlab.com/ee/ci/mobile_devops.html).\n\n\nGitLab Mobile DevOps is a collection of features built right into GitLab to\nsolve the biggest challenges mobile teams face in establishing a DevOps\npractice.\n\n\nIn this blog post, I’ll demonstrate how to set up an automated CI/CD\npipeline using GitLab and [fastlane](https://fastlane.tools/).\n\n\n## Prerequisites\n\nTo get started, there are a few prerequisites you’ll need:\n\n\n* An Apple Developer account -\n[https://developer.apple.com/](https://developer.apple.com/)\n\n* Ruby and XCode command line tools installed on your local machine\n[https://docs.fastlane.tools/getting-started/ios/setup](https://docs.fastlane.tools/getting-started/ios/setup/) \n\n\n> Try out our [Android CI/CD with GitLab\ntutorial](/blog/android-cicd-with-gitlab/).\n\n\n## Reference project\n\nFor this walkthrough, we’ll use the iOS demo project for reference:\n[https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo)\n\n\n## Install fastlane\n\nIf you haven’t done so yet, the first step will be to install fastlane. Do\nthis by creating a file in the root of your project called Gemfile. Give it\nthe following contents:\n\n\n```\n\nsource \"https://rubygems.org\"\n\n\ngem \"fastlane\"\n\n```\n\n\nThen, from the terminal in your project, run:\n\n\n```\n\nbundle install\n\n```\n\n\nThis command will install fastlane, and all of its related dependencies.\n\n\n## Initialize fastlane\n\nNow that fastlane is installed, we can set it up for our project. Run the\nfollowing command from the terminal in your project and choose Option No. 2\nsince we will be targeting Test Flight in this tutorial:\n\n\n```\n\nbundle exec fastlane init\n\n```\n\n\nRunning this command will create a new folder in your project called\n`fastlane`. This folder will contain two files `Appfile` and `Fastfile`. \n\n\n![Initialize\nFastlane](https://about.gitlab.com/images/blogimages/2023-04-15-ios-cicd-with-gitlab/fastlane-init.png)\n\n\nThe Appfile contains the configuration information for the app, and the\nFastfile has some sample code that we will replace later. See the fastlane\ndocs for more information about the configuration details in the Appfile\n[https://docs.fastlane.tools/advanced/Appfile/](https://docs.fastlane.tools/advanced/Appfile/)\n\n\n## Initialize fastlane match\n\nThe next step will be to set up fastlane Match, which is the part of\nfastlane that handles code signing. For more information on fastlane match,\nsee the docs\n[https://docs.fastlane.tools/actions/match/](https://docs.fastlane.tools/actions/match/\n)\n\n\nWe’ll start by running the following command from the terminal in your\nproject:\n\n\n```\n\nbundle exec fastlane match init\n\n```\n\n\nThis command will prompt you to choose which storage backend you want to use\n(select gitlab_secure_files) and to input your project path (for example:\ngitlab-org/gitlab). It will then generate a fastlane Matchfile configured to\nuse your project as the storage backend for fastlane Match.\n\n\n![Initialize fastlane\nMatch](https://about.gitlab.com/images/blogimages/2023-04-15-ios-cicd-with-gitlab/match-init.png)\n\n\n## Generate a project access token\n\nNext, you'll need a GitLab Access Token to use fastlane Match from your\nlocal machine. To create a project access token, visit the Access Tokens\nsection under Settings in your GitLab project. Create a new token with\nmaintainer access to the “api” scope.\n\n\nThen run the following command from the terminal in your project replacing\n“YOUR_NEW_TOKEN” with the access token you just generated:\n\n\n```\n\nexport PRIVATE_TOKEN=YOUR_NEW_TOKEN\n\n```\n\n\nThis will configure fastlane to use this access token when making fastlane\nMatch requests to your project.\n\n\n## Generate signing certificates\n\nNow that fastlane Match is configured, we can use it to generate the signing\ncertificates and provisioning profiles for our app and upload them to\nGitLab.\n\n\nNOTE: If you already have these files for your app, see the instructions in\nthis blog post on how to use fastlane to import your existing code signing\nfiles\n[/blog/mobile-devops-with-gitlab-part-3-code-signing-for-ios-with-gitlab-and-fastlane/](/blog/mobile-devops-with-gitlab-part-3-code-signing-for-ios-with-gitlab-and-fastlane/).\n\n\nRun the following command from the terminal in your project to generate\ndevelopment code signing files and upload them to GitLab.\n\n\n```\n\nbundle exec fastlane match development\n\n```\n\n\nWhen this command completes, go to the CI/CD settings page in your project\nand scroll down to the Secure Files section to see the files that were just\ngenerated and added to your project.\n\n\nWhile we’re here, we can go ahead and do that same thing for the appstore\ncode signing files. Run the following command to generate the appstore code\nsigning files and upload them to GitLab.\n\n\n```\n\nbundle exec fastlane match appstore\n\n```\n\n\n## Update Xcode configuration\n\nWith the code signing files ready to go, we have one small change to make in\nXcode. In your project in Xcode, go to the Signing & Capabilities section\nand disable automatically managing code signing. Then, select the\nappropriate provisioning profile and signing certificate from the list based\non your build target. The certificates we just generated will show up in\nthat list.\n\n\n![Configure Xcode Provisioning\nProfiles](https://about.gitlab.com/images/blogimages/2023-04-15-ios-cicd-with-gitlab/xcode.png)\n\n\nWith all of our code signing configuration in place, we can now move on to\nsetting up the integration with the Apple App Store.\n\n\n## Apple App Store integration\n\nThe final bit of configuration is the Apple App Store integration. To do\nthis, we’ll need to create an API key in App Store Connect. See the\ninstructions here to create and download the key file to your location\nmachine. This key should have the role of App Manager.\n[https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api](https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api)\n\n\nOnce the key is generated, go to Settings, Integrations in your project, and\nclick on the integration for Apple App Store Connect. You’ll be asked to\nsupply the issuer ID and key ID from App Store Connect, along with the key\nfile you just downloaded. With all of that configuration in place, click the\nTest Settings button to ensure everything works. If it gives you an error,\ndouble check your settings and try again. Once it’s working, click Save\nChanges to save and activate the integration. \n\n\nWith the integration activated, the following CI variables are added to all\npipelines on protected branches and tags:\n\n\n* `APP_STORE_CONNECT_API_KEY_ISSUER_ID`\n\n* `APP_STORE_CONNECT_API_KEY_KEY_ID`\n\n* `APP_STORE_CONNECT_API_KEY_KEY`\n\n\nThese CI variables can be used by fastlane or any custom tooling to interact\nwith the Apple App Store to upload builds, or perform other API enabled\ntasks.\n\n\n## Fastfile\n\nWith all of our configuration in place, we can now drop in a sample Fastfile\nto show how to perform the build, sign, and release actions.\n\n\nFrom the [sample\nproject](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo),\ncopy the contents of the fastlane/Fastfile and paste it into the Fastfile in\nyour project, replacing the existing content. \n\n\n[https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo/-/blob/main/fastlane/Fastfile](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo/-/blob/main/fastlane/Fastfile)\n\n\nThis sample Fastfile contains two lanes, which are actions fastlane can\nexecute. The lanes in this file are `build` and `beta`. \n\n\n### Build\n\nThe build lane will perform just a couple of actions to `setup_ci`, `match`,\nand `build_app`. This will use the development certificate we generated with\nfastlane Match earlier to build and sign the app for development. \n\n\n### Beta\n\nThe beta lane takes a few more steps to `setup_ci`, `match`,\n`app_store_connect_api_key`, `increment_build_number`, `build_app`, and\n`upload_to_testflight`. This lane will use the appstore certificates we\ngenerated with faslane Match earlier to build and sign the app for an\nappstore release. This lane also uses the App Store Connect integration to\nconnect to the app store to determine the next build number to use, and to\nupload the final build to Test Flight. \n\n\n### .gitlab-ci.yml\n\nWith the fastlane configuration ready to go, the last step is to hook it up\nto GitLab CI. \n\n\nFrom the [sample\nproject](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo),\ncopy the contents of the `.gitlab-ci.yml` file and paste it into the\nproject. \n\n\n[https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo/-/blob/main/.gitlab-ci.yml](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo/-/blob/main/.gitlab-ci.yml\n)\n\n\nThis is a simplified CI configuration that created two CI jobs to run each\nof the lanes in fastlane on the GitLab macOS shared runners. The build job\nwill run for all CI pipelines and the beta job will only be run on CI\npipelines on the master branch. The beta job is also manually triggered, so\nyou can control when the beta release is pushed to Test Flight. \n\n\nWith all of this in place, commit all of these changes and push them up to\nyour project. The CI pipeline will kick off, and you can see these jobs in\naction. \n\n\nCover image by \u003Ca\nhref=\"https://unsplash.com/@john_cameron?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">John\nCameron\u003C/a> on \u003Ca\nhref=\"https://unsplash.com/photos/DgRb7aAGK4k?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">Unsplash\u003C/a>\n\n{: .note}\n",[789,108,9],{"slug":3131,"featured":6,"template":700},"ios-cicd-with-gitlab","content:en-us:blog:ios-cicd-with-gitlab.yml","Ios Cicd With Gitlab","en-us/blog/ios-cicd-with-gitlab.yml","en-us/blog/ios-cicd-with-gitlab",{"_path":3137,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3138,"content":3144,"config":3149,"_id":3151,"_type":13,"title":3152,"_source":15,"_file":3153,"_stem":3154,"_extension":18},"/en-us/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment",{"title":3139,"description":3140,"ogTitle":3139,"ogDescription":3140,"noIndex":6,"ogImage":3141,"ogUrl":3142,"ogSiteName":685,"ogType":686,"canonicalUrls":3142,"schema":3143},"Jenkins to GitLab: The ultimate guide to modernizing your CI/CD environment","Learn how to migrate from Jenkins to the integrated CI/CD of the GitLab DevSecOps Platform to deliver high-quality software rapidly.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663000/Blog/Hero%20Images/tanukilifecycle.png","https://about.gitlab.com/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Jenkins to GitLab: The ultimate guide to modernizing your CI/CD environment\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2023-11-01\",\n      }",{"title":3139,"description":3140,"authors":3145,"heroImage":3141,"date":3146,"body":3147,"category":1119,"tags":3148},[1096],"2023-11-01","\nIn today's dynamic landscape of software development, certain requirements have become paramount for delivering high-quality software rapidly. These requirements include the need for cloud compatibility, faster development cycles, improved collaboration, containerization, enhanced development experiences, and the integration of AI-driven capabilities for better efficiency and speed. Jenkins, a longstanding and respected continuous integration (CI) tool, has admirably played a role in many teams' software development for years. However, as more teams adopt DevOps/DevSecOps strategies for their software delivery, leveraging the integrated CI that is available in a DevSecOps platform like GitLab can provide benefits that Jenkins does not. \n\nSome organizations find themselves hesitating to migrate, not because they doubt the benefits of a top-tier [CI/CD](https://about.gitlab.com/topics/ci-cd/) solution such as GitLab, but due to the complexities of their existing Jenkins implementations. It's understandable that such a transition can seem daunting. \n\nIn this blog, you'll find several migration strategies to help transition from Jenkins to GitLab and make the process smoother and more manageable.\n\n## Migrating to GitLab\nIt's become evident that for organizations seeking a CI/CD solution that can seamlessly support their evolving demands, GitLab emerges as a powerful game-changer. Let's explore why transitioning to this advanced platform is transformative for Jenkins users.\n\n### Why migrate to GitLab \nBefore we delve into the migration approaches, let's take a moment to understand GitLab CI and what makes it a compelling choice for modern CI/CD needs.\n\n> Try GitLab CI/CD today with [a free trial of Ultimate](https://gitlab.com/-/trials/new).\n\n### GitLab CI overview\nGitLab CI is an integral part of the GitLab [AI-powered](https://about.gitlab.com/gitlab-duo/) DevSecOps Platform, which offers a comprehensive and unified solution for DevSecOps and CI/CD. GitLab's design revolves around streamlining development workflows, fostering collaboration, enhancing security, and ensuring scalability.\n\n### Key features of GitLab CI\nThese are the key features of GitLab CI:\n- **Unified platform:** GitLab CI is more than just a CI/CD tool; it's part of a broader ecosystem that includes source code management, project management, security features, analytics and more. This unified platform streamlines workflows and enhances collaboration among development teams.\n- **Containerization and orchestration:** GitLab CI/CD is designed with containerization in mind, offering native support for Docker and Kubernetes. This enables seamless integration of container technologies into your CI/CD pipelines.\n- **Security by design:** Security is a top priority, and GitLab CI incorporates features such as static code analysis and vulnerability scanning to help teams identify and address security issues early in the development process.\n- **GitOps principles:** GitLab CI aligns with [GitOps principles](https://about.gitlab.com/blog/the-ultimate-guide-to-gitops-with-gitlab/), emphasizing version-controlled, declarative configurations for infrastructure and application deployments. This approach enhances the reliability and repeatability of deployments.\n\nGet familiar with GitLab CI with this tutorial:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/WKR-7clknsA?si=T21Fe10Oa0rQ0SGB\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nWith that understanding of GitLab CI's capabilities, let's explore the migration steps and strategies for Jenkins users looking to leverage the benefits of GitLab CI.\n\n## A recommended step-by-step Jenkins-to-GitLab CI migration\nWhen considering a migration from Jenkins to GitLab CI, we strongly recommend following a well-structured, step-by-step approach to ensure a seamless transition. Here's our recommended process:\n1. **Pipeline assessment:** Start by conducting a comprehensive inventory of all your existing pipelines in Jenkins. This initial step will help you gain a clear understanding of the scope and complexity of the migration.\n2. **Parallel migration:** Begin the migration process by selecting individual pipelines and moving them to GitLab CI one at a time. Continue to maintain the use of Jenkins for your ongoing work during this transition to minimize disruptions.\n3. **Code verification:** We advise beginning with verification checks in CI. Run both the Jenkins and GitLab CI pipelines in parallel. This dual approach allows you to directly compare the two workflows and identify any issues in the new GitLab workflows. During this phase, keep the GitLab workflow as an optional choice while Jenkins remains required.\n4. **Continuous validation:** After running both pipelines in parallel for a full iteration, thoroughly evaluate the outcomes from each pipeline. This evaluation should consider various factors, including status codes, logs, and performance. \n5. **GitLab CI transition:** As you gain confidence in the reliability and effectiveness of GitLab CI through the parallel runs, make the transition to the GitLab CI workflow as the required standard while Jenkins continues to operate in the background.\n6. **Jenkins phaseout:** After a second iteration, when you are confident in the performance and stability of GitLab CI, you can begin to remove the Jenkins job from your code verification pipeline. This successful transition will enable you to retire Jenkins from this particular aspect of your CI/CD process.\n\nThis recommended approach ensures that your migration is a gradual evolution, allowing you to identify and address any issues or discrepancies before fully committing to GitLab CI. Running Jenkins and GitLab CI pipelines in parallel provides valuable insights and ensures the effective streamlining of your CI/CD processes.\n\n## Preparing for migration: Training and communication\nTo ensure a smooth and successful migration from Jenkins to GitLab CI, follow these essential steps:\n- **Stakeholder communication:** Start by announcing your migration plans and timelines to all relevant stakeholders. This includes DevOps teams, developers, and QA engineers. Transparency in communication is crucial to ensure that everyone understands the objectives and expectations of the migration.\n- **Knowledge-level training:** Conduct knowledge-level training sessions for your teams to promote GitLab CI adoption.\nCover topics such as using GitLab CI, understanding the YAML syntax, and how to create a basic pipeline.\nProvide team members with the knowledge and skills necessary to navigate the new GitLab CI environment effectively.\n- **Hands-on learning:** Encourage hands-on learning by pairing up developers.\nCreate opportunities for them to learn from each other's experiences throughout the migration process.\n\nBy following these instructions for training and communication, you'll build a strong foundation for a successful migration, empowering your teams to adapt and thrive in the new environment.\n\n## 3 Jenkins-to-GitLab CI migration strategies\nThere are different strategies to consider. These three strategies offer flexibility, allowing organizations to choose the path that best aligns with their specific needs and resources. Let's explore these strategies in detail to help you make an informed decision about which one suits your organization best.\n\n### Migration Strategy 1: Using GitLab CI for new projects\nThe first migration strategy involves a gradual transition. While you maintain your existing Jenkins infrastructure for ongoing projects, you introduce GitLab CI for new projects. This approach allows you to harness the modern features of GitLab CI without disrupting your current work.\n\n#### Benefits of Migration Strategy 1\nThe benefits of this approach include the following:\n- New projects can leverage GitLab CI's advanced features right from the start. \n- This strategy minimizes the risk of disrupting existing workflows, as your existing Jenkins setup remains intact.\n- Your team can gradually adapt to GitLab CI, building confidence and expertise without the pressure of an immediate full-scale migration.\n\n#### Challenges of Migration Strategy 1\nThe challenges of this approach include the following:\n- Operating two CI/CD platforms simultaneously can introduce complexity, especially in terms of integration and team collaboration.\n- Managing projects on different platforms may require careful coordination to ensure consistency in processes and security practices.\n\nThis strategy offers a smooth and manageable transition by allowing you to harness GitLab CI's strengths for new projects, while your existing Jenkins infrastructure continues to support ongoing work.\n\n### Migration Strategy 2: Migrating only strategic projects\nIn this strategy, you identify specific projects within your organization that stand to benefit the most from the capabilities of GitLab CI. Instead of preparing for a wholesale migration, you start by focusing your efforts on migrating these strategically selected projects first.\n\n#### Benefits of Migration Strategy 2\nThe benefits of this approach include the following:\n- By concentrating on key projects, you can realize significant improvements in those areas where GitLab CI aligns with specific needs.\n- This approach reduces the complexity of migrating everything at once, minimizing the potential for disruptions.\n- You can gradually build confidence with GitLab CI and its benefits before considering further migrations.\n\n#### Challenges of Migration Strategy 2\nThe challenges of this approach include the following:\n- Even though you're not migrating all projects, the chosen projects' migration can still be intricate and require careful planning.\n- Ensuring seamless collaboration between projects on different platforms may require additional attention.\n\nThis strategy allows you to maximize the impact of GitLab CI by focusing on strategic areas, minimizing risk, and gradually gaining experience with the new tool.\n\n### Migration Strategy 3: Migrating everything\nThe third strategy is a comprehensive migration where you commit to moving all your CI/CD processes, projects, and workflows to GitLab CI. This approach aims for uniformity and simplification of CI/CD across all projects. This strategy can benefit from taking an iterative approach. Consider starting with new projects, followed by migrating strategic projects, and then leverage your growing knowledge and experience with GitLab CI to complete the migration of remaining projects. \n\n#### Benefits of Migration Strategy 3\nThe benefits of this approach include the following:\n- Uniform CI/CD processes across all projects can streamline administration and maintenance, reducing complexity.\n- You can take full advantage of GitLab CI's modern capabilities, from Infrastructure as Code to enhanced security features.\n- As your projects grow, GitLab CI is designed to handle increased demands, ensuring long-term scalability.\n\n#### Challenges of Migration Strategy 3\nThe challenges of this approach include the following:\n- A full-scale migration can be intricate, requiring meticulous planning and implementation.\n- The transition may disrupt ongoing projects and require a significant time investment.\n- Investment in training and potential tool migration expenses should be considered.\n\nOpt for this approach if uniformity and consolidation of CI/CD processes are a high priority, and you have the resources to execute a full migration.\n\nThe migration strategy you select should align with your organization's specific needs and circumstances. In all cases, the ultimate goal is to enhance your development process with modern CI/CD tools like GitLab CI, which offers scalability, infrastructure automation, security, and collaboration features that align with today's development needs.\n\n## Technical insights: How the migration works\nMoving your CI/CD workflows from Jenkins to GitLab CI is a transformative journey, and understanding how it works is vital for a successful transition.\n\n### Understanding the configurations: Jenkinsfile vs. .gitlab-ci.yml\nThe heart of your CI/CD pipeline lies in the configurations defined in your Jenkinsfile (for Jenkins) and .gitlab-ci.yml (for GitLab CI). While there are some similarities between these configuration files, there are notable differences as well.\n\n#### Similarities\n- Both files define the stages, jobs, and steps of your CI/CD process.\n- You specify the desired build, test, and deployment steps in both files.\n- Environment variables and settings can be configured in either file.\n\n#### Differences\n- Jenkinsfile uses Groovy for scripting, while .gitlab-ci.yml uses YAML. This change in language affects the way you write and structure your configurations.\n- The process of defining pipelines is more intuitive in .gitlab-ci.yml, with a cleaner, more human-readable syntax.\n- GitLab CI provides a wide range of built-in templates and predefined jobs, simplifying configuration and reducing the need for custom scripting.\n\n### Manually converting the pipeline configuration\nCurrently, migrating your existing Jenkins pipelines to GitLab CI is typically done manually. This means analyzing your Jenkinsfile and re-creating the equivalent configurations in .gitlab-ci.yml. While there are similarities in the concepts and structure, the differences in syntax and the specific capabilities of each platform require careful consideration during the migration.\n\n## Strategic planning for a smooth transition\nMigrating from Jenkins to GitLab CI requires meticulous planning to ensure a seamless transition. It's crucial to assess the disparities between the two systems and evaluate their impact on your workflow, considering aspects like security, cost, time, and capacity.\n\nOnce you've identified these differences and devised your migration strategy, break down the migration into key steps. These include setting up GitLab CI pipelines, securely transferring data from Jenkins to GitLab CI, and integrating GitLab CI into your existing tools and processes. \n\n## Case study: A seamless transition for Lockheed Martin\nLet's look at a real-world case study to illustrate the effectiveness of the \"Migrate Everything\" strategy. [Lockheed Martin](https://about.gitlab.com/customers/lockheed-martin/), the world’s largest defense contractor, had been using Jenkins for several years. As their project portfolio expanded, they realized that their Jenkins implementation with a wide variety of DevOps tools was becoming increasingly complex to manage. They were also eager to adopt modern CI/CD capabilities that Jenkins struggled to provide.\n\nIn collaboration with GitLab, Lockheed Martin decided to undertake a comprehensive migration to GitLab CI. Their goals included achieving consistency in their CI/CD processes, simplifying administration and maintenance, and taking full advantage of The GitLab Platform’s robust features.\n\nThe comprehensive migration strategy proved to be a resounding success for Lockheed Martin. With GitLab CI, they not only streamlined their CI/CD processes but achieved remarkable results. **They managed to run CI pipeline builds a staggering 80 times faster, retired thousands of Jenkins servers, and reduced the time spent on system maintenance by a staggering 90%. This monumental shift resulted in a significant increase in efficiency and productivity for Lockheed Martin.**\n\nThis case study showcases how a comprehensive migration strategy can be effective for organizations looking to leverage GitLab capabilities across all their projects.\n\nFor more in-depth insights into Lockheed Martin's successful transition to GitLab and how it streamlined their software development processes, check out [the detailed case study](https://about.gitlab.com/customers/lockheed-martin/).\n\n## GitLab documentation and support\nFor those embarking on this migration journey, GitLab offers documentation to guide you through the process. You can find valuable resources in GitLab's [official documentation](https://docs.gitlab.com/ee/ci/migration/jenkins.html).\n\nIn addition to documentation, GitLab's Professional Services team is available to assist organizations in their migrations. They bring expertise and experience to ensure a smooth transition. Whether it's understanding the nuances of Jenkinsfile to .gitlab-ci.yml conversion or optimizing your CI/CD workflows, their support can be invaluable.\n\n> Try GitLab CI/CD today with [a free trial of Ultimate](https://gitlab.com/-/trials/new).\n",[9,278,720,232,851,789],{"slug":3150,"featured":6,"template":700},"jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment","content:en-us:blog:jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment.yml","Jenkins Gitlab Ultimate Guide To Modernizing Cicd Environment","en-us/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment.yml","en-us/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment",{"_path":3156,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3157,"content":3163,"config":3169,"_id":3171,"_type":13,"title":3172,"_source":15,"_file":3173,"_stem":3174,"_extension":18},"/en-us/blog/keep-git-history-clean-with-interactive-rebase",{"title":3158,"description":3159,"ogTitle":3158,"ogDescription":3159,"noIndex":6,"ogImage":3160,"ogUrl":3161,"ogSiteName":685,"ogType":686,"canonicalUrls":3161,"schema":3162},"How to keep your Git history clean with interactive rebase","Interactive rebase is one of Git’s most versatile tools. Here's how to use it to correct commit messages, fix mistakes, and more.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662593/Blog/Hero%20Images/title-image.png","https://about.gitlab.com/blog/keep-git-history-clean-with-interactive-rebase","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to keep your Git history clean with interactive rebase\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tobias Günther\"}],\n        \"datePublished\": \"2020-11-23\",\n      }",{"title":3158,"description":3159,"authors":3164,"heroImage":3160,"date":3166,"body":3167,"category":785,"tags":3168},[3165],"Tobias Günther","2020-11-23","## What is interactive rebase? \n\n\nInteractive [rebase](/solutions/source-code-management/), or\nGit rebase interactive, is sometimes called the \"Swiss Army Knife\" of Git –\nbecause it contains so many different tools, for so many different use\ncases! However, there's one main, overarching use case: _cleaning up your\nlocal commit history_.\n\n\nMind the word \"local\": it should only be used for cleaning up your own,\nlocal commit history, for example before integrating one of your feature\nbranches into a team branch. In contrast, it should NOT be used on commit\nhistory that has already been pushed and shared on a remote repository.\nInteractive rebase is one of those tools that \"rewrite\" Git history – and\nyou shouldn't do this on commits that have already been shared with others.\n\n\nWith this little warning message out of the way, let's look at some\npractical examples! \n\n\nNote: for easier visualization of the scenarios and workflows in this post,\nI’ve been using the [\"Tower\" Git desktop\nGUI](https://www.git-tower.com/?utm_source=gitlab&utm_medium=guestpost&utm_campaign=interactive-rebase)\nin some of my screenshots.\n\n{: .note}\n\n\n## Correcting an old commit message with Git rebase interactive\n\n\nSometimes you notice a typo in an **old commit message** – or you've\nforgotten to mention something in the description that is noteworthy. If we\nwere talking about the _very last_ commit, we could have simply used the\n`--amend` option of the `git commit` command. But for older commits you will\nhave to use interactive rebase to change them after the fact.\n\n\nHere's an example of a commit message gone horribly wrong that we want to\ncorrect:\n\n\n![A bad commit message that needs\ncorrection](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/bad-commit-message@2x.png){:\n.shadow.medium.center}\n\nA bad commit message that needs correction\n\n{: .note.text-center}\n\n\nThe first step in _any_ Git interactive rebase session is to **determine\nwhat part of commit history you want to manipulate**. To again take the\nabove example: in order to change this bad commit we have to start the\nsession at its _parent_ commit.\n\n\n![Starting our interactive rebase\nsession](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/start-at-parent-commit@2x.png){:\n.shadow.medium.center}\n\nStarting our interactive rebase session\n\n{: .note.text-center}\n\n\nWe can now feed this starting commit's hash to the Git rebase interactive\ncommand:\n\n\n```\n\n$ git rebase -i 0023cddd\n\n```\n\n\nAn editor window will now open, containing a list of the commits that you\njust selected for manipulation. And don't be surprised because they are in\n_reverse order_: in an interactive rebase session, Git will reapply the old\ncommits, item after item – which means that reversing the order is correct\nfrom Git's perspective.\n\n\n![Editor window with the selected\ncommits](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/editor-window-start-ir@2x.png){:\n.shadow.medium.center}\n\nEditor window with the selected commits\n\n{: .note.text-center}\n\n\nOne other important thing to note about this editor window: _you don't\nperform the actual manipulations here_! Or, in this concrete example, you do\nNOT go ahead and change the commit message here. Instead, you only mark the\ncommit you want to change with an action keyword. In our case, because we\nwant to change a commit’s message, we mark the line with \"reword\". If you\nthen save and close this editor window, a new one will open, containing the\nold commit’s message. Now is the time to finally make your changes:\n\n\n![Finally, we can make our\nchanges](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/correct-commit-message.gif){:\n.shadow.medium.center}\n\nFinally, we can make our changes\n\n{: .note.text-center}\n\n\nAfter saving and closing once more, the interactive rebase session is\ncomplete and our old commit message has been corrected!\n\n\n## Combining multiple commits into one using interactive rebase\n\n\nAnother use case for interactive rebase is when you want to **combine\nmultiple old commits into one**. Although, of course, the golden rule of\nversion control applies: in most situations, it's beneficial to create more\nand smaller commits instead of a few big ones. However, as with everything,\nwe might find that we have overdone this and now want to meld two or more\nold commits into a single one.\n\n\nTo make a concrete example, let's say we want to combine the following\nselected commits into a single one:\n\n\n![Let's combine multiple commits into\none](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/squash-selected-commits@2x.png){:\n.shadow.medium.center}\n\nLet's combine multiple commits into one\n\n{: .note.text-center}\n\n\nJust like in our first case, we begin by starting the interactive rebase\nsession at least at the parent commit of the one we want to manipulate.\n\n\n```\n\n$ git rebase -i 2b504bee\n\n```\n\n\nAgain, an editor window will open, listing that part of our commit history\nthat we want to manipulate:\n\n\n![Marking lines with\n\"squash\"](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/squash-mark-commit@2x.png){:\n.shadow.medium.center}\n\nMarking lines with \"squash\"\n\n{: .note.text-center}\n\n\nThe action keyword we are going to use here is called \"squash.\" And there's\nonly one important piece of information you need to know about squash in\norder to use it: _the line we mark up with the \"squash\" keyword will be\ncombined with the line directly above_. That’s why, as you can see in my\nscreenshot above, I’ve marked line #2 with \"squash\" in order to combine it\nwith line #1.\n\n\nWe can now save and close the editor window and again watch and a new window\nappear: we are now asked to provide a commit message for the new commit that\nis created when combining those two old ones.\n\n\n![Entering a new message for the new, squashed\ncommit](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/squash-enter-new-message@2x.png){:\n.shadow.medium.center}\n\nEntering a new message for the new, squashed commit\n\n{: .note.text-center}\n\n\nAfter saving and closing this editor window, you will see that a new commit\nwas created that contains the changesets of both old commits. Voila!\n\n\n## Fixing a mistake with interactive rebase\n\n\nAnother use case for interactive rebase is when you found a mistake in one\nof your earlier commits. And it doesn't matter what exactly you messed up:\nyou could have forgotten to add a certain change, should have deleted a\nfile, or simply introduced a typo...\n\n\nThe natural tendency, in such a situation, is to simply create a new commit\nthat corrects the mistake. But on the other hand, this will mess up our\ncommit history: making an original commit, and then adding a \"band-aid\"\ncommit just to fix some mistakes… that’s a messy way of working. Your commit\nhistory will soon become hard to understand, because it's littered with all\nthose little \"quick fix commits\"!\n\n\nThis is where \"fixup,\" one of the tools that come with interactive rebase,\ncomes in very handy. Fixup takes this \"quick fix\" commit, applies its\nchanges to the original commit (thereby correcting it), and then gets rid of\nthe band-aid commit:\n\n\n![How \"fixup\"\nworks](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/diagram-fixup.png){:\n.medium.center}\n\nHow \"fixup\" works\n\n{: .note.text-center}\n\n\nAfter we're done, it looks as if there had never been a problem with our\noriginal commit! So let's walk through this using a practical example. \n\n\nThe first step is to do whatever is necessary to fix the problem: this could\nmean adding a new file, making changes to existing ones, deleting obsolete\nfiles... you \"just\" need to produce the changes that correct the mistake.\n\n\nThe next step is to commit these changes to the repository – but with a\nlittle extra: when making the commit, we are going to use the `--fixup` flag\nand tell Git the commit hash of our bad commit:\n\n\n```\n\n$ git add corrections.txt\n\n$ git commit --fixup 2b504bee\n\n```\n\n\nWhen you now take a look at the commit history, you will see that a pretty\nordinarily looking commit has been created – probably not the magic and\nfireworks you would have expected. But if you take a closer look, you will\nsee that something’s going on: the new commit has automatically been\nprepended with \"fixup !\" and the commit subject of our bad commit.\n\n\n![The original commit and the fix\ncommit](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/fixup_create-fix-commit@2x.png){:\n.shadow.medium.center}\n\nThe original commit and the fix commit\n\n{: .note.text-center}\n\n\nThe third step now is to start the interactive rebase session. Again, we\nchoose the parent of our bad commit as the starting point...\n\n\n```\n\n$ git rebase -i 0023cddd --autosquash\n\n```\n\n\n... and as the second part of the secret sauce, we are using the\n`--autosquash` flag. This option makes sure that we don't have to do\n_anything_ in the editor window that is now open. Take a close look at the\nsituation:\n\n\n![Our fix commit is marked \"fixup\" and sorted to the right\nposition](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/fixup_editor@2x.png){:\n.shadow.medium.center}\n\nOur fix commit is marked \"fixup\" and sorted to the right position\n\n{: .note.text-center}\n\n\nYou will see that Git automatically did two things for us:\n\n1. It marked our band-aid commit as \"fixup.\"\n\n2. It re-ordered the lines so that our band-aid commit appears directly\nbelow our bad commit. This is because fixup works exactly like squash in\nthat it _combines with the line above_.\n\n\nIn other words: there's nothing left to do for us but save and close the\neditor window.\n\n\nLet's take another look at the commit history:\n\n\n![A happy\nending!](https://about.gitlab.com/images/blogimages/how-to-keep-your-git-history-clean-with-interactive-rebase/fixup_final-corrected@2x.png){:\n.shadow.medium.center}\n\nA happy ending!\n\n{: .note.text-center}\n\n\nNot only does our originally bad commit now contain the changes from our\nband-aid commit. But on top of that, the ugly band-aid commit has\ndisappeared from the commit history! Everything is nice and clean, just as\nif there had never been a problem!\n\n\n## Discover the power of Git rebase interactive\n\n\nThere are lots of use cases for interactive rebase – and most of them in the\ndepartment of “fixing mistakes”. For an overview of other useful things you\ncan do, I recommend the _free_ [\"First Aid Kit for\nGit\"](https://www.git-tower.com/learn/git/first-aid-kit?utm_source=gitlab&utm_medium=guestpost&utm_campaign=interactive-rebase):\nit’s a collection of short videos (2-3 min per episode) that help you learn\nto undo mistakes using interactive rebase and other Git tools.\n\n\nEditor's note: I had to use interactive rebase when reviewing this very\npost! One of my commits included an image that was greater than 1MB which is\nagainst the rules for GitLab website project. I had to go back and fix that\ncommit to include a correctly sized image instead. Thanks for the lesson,\nuniverse! 😁\n\n{: .note}\n\n\n## More Git tips and tricks\n\n\n- [15 Git tips to improve your\nworkflow](/blog/15-git-tips-improve-workflow/)\n\n- [How Git Partial Clone lets you fetch only the large file you\nneed](/blog/partial-clone-for-massive-repositories/)\n\n- [Git happens! 6 Common Git mistakes and how to fix\nthem](/blog/git-happens/)\n\n\n### About the guest author\n\n\n_[Tobias Günther](https://twitter.com/gntr) is the CEO of\n[Tower](https://www.git-tower.com/?utm_source=gitlab&utm_medium=guestpost&utm_campaign=interactive-rebase),\nthe popular Git desktop client that helps more than 100,000 developers\naround the world to be more productive with Git._\n\n\nCover image by [David Taljat](https://www.pexels.com/@david-taljat-3748658)\non\n[Pexels](https://www.pexels.com/photo/yellow-and-blue-line-on-gray-asphalt-road-5690623/)\n\n{: .note}\n",[742,9],{"slug":3170,"featured":6,"template":700},"keep-git-history-clean-with-interactive-rebase","content:en-us:blog:keep-git-history-clean-with-interactive-rebase.yml","Keep Git History Clean With Interactive Rebase","en-us/blog/keep-git-history-clean-with-interactive-rebase.yml","en-us/blog/keep-git-history-clean-with-interactive-rebase",{"_path":3176,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3177,"content":3183,"config":3189,"_id":3191,"_type":13,"title":3192,"_source":15,"_file":3193,"_stem":3194,"_extension":18},"/en-us/blog/kubernetes-overview-operate-cluster-data-on-the-frontend",{"title":3178,"description":3179,"ogTitle":3178,"ogDescription":3179,"noIndex":6,"ogImage":3180,"ogUrl":3181,"ogSiteName":685,"ogType":686,"canonicalUrls":3181,"schema":3182},"Kubernetes overview: Operate cluster data on the frontend","GitLab offers a built-in solution for monitoring your Kubernetes cluster health. Learn more about the technical design and functionality with this detailed guide.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099045/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2816%29_3L7ZP4GxJrShu6qImuS4Wo_1750099045397.png","https://about.gitlab.com/blog/kubernetes-overview-operate-cluster-data-on-the-frontend","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Kubernetes overview: Operate cluster data on the frontend\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Anna Vovchenko\"}],\n        \"datePublished\": \"2024-06-20\",\n      }",{"title":3178,"description":3179,"authors":3184,"heroImage":3180,"date":3186,"updatedDate":1446,"body":3187,"category":785,"tags":3188},[3185],"Anna Vovchenko","2024-06-20","Accessing real-time cluster information is crucial for verifying successful\nsoftware deployments and initiating troubleshooting processes. In this\narticle, you'll learn about GitLab's enhanced Kubernetes integration,\nincluding how to leverage the Watch API for real-time insights into\ndeployment statuses and streamlined troubleshooting capabilities. \n\n\n## What are GitLab's Kubernetes resources?\n\n\nGitLab offers a dedicated [dashboard for\nKubernetes](https://gitlab.com/groups/gitlab-org/-/epics/2493 \"Visualize the\ncluster state in GitLab\") to understand the status of connected clusters\nwith an intuitive visual interface. It is integrated into the Environment\nDetails page and shows resources relevant to the environment. Currently,\nthree types of Kubernetes resources are available:\n\n\n- pods filtered by the Kubernetes namespace\n\n- services\n\n- Flux resource\n([HelmRelease](https://fluxcd.io/flux/components/helm/helmreleases/) or\n[Kustomization](https://fluxcd.io/flux/components/kustomize/kustomizations/))\n\n\nFor these resources, we provide general information, such as name, status,\nnamespace, age, etc. It is represented similarly to what the\n[kubectl](https://kubernetes.io/docs/reference/kubectl/) command would show\nwhen run from the Kubernetes cluster. More details can be found when\nclicking each resource. The side drawer shows the list of labels, annotations,\ndetailed status, and spec information presented as read-only YAML code blocks,\nand streams real-time Kubernetes events filtered for that specific resource.\nFor pods, users can navigate to a dedicated logs view where they select a\ncontainer and stream its real-time logs, providing crucial debugging information.\n\n\nThe information provided helps to visualize the cluster state, spot any\nissues, and debug problematic deployments right away. Users can also take\nimmediate action — deleting failed pods, triggering Flux reconciliation, or\nsuspending/resuming synchronization — all from the same interface without\ncontext switching to command-line tools.\n\n\n## Frontend to cluster communication: The GitLab solution\n\n\nWe have developed a range of tools and solutions to enable a seamless\nconnection and management of Kubernetes clusters within GitLab. One of the\ncore components of this system is the [GitLab agent for\nKubernetes](https://docs.gitlab.com/ee/user/clusters/agent/install/). This\npowerful tool provides a secure bidirectional connection between a GitLab\ninstance and a Kubernetes cluster. It is composed of two main components:\n**agentk** and **KAS** (Kubernetes agent server).\n\n\n![Kubernetes flow\nchart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099055229.png)\n\n\nagentk is a lightweight cluster-side component. It is responsible for\nestablishing a connection to a KAS instance and waiting for requests to\nprocess. It is proxying requests from KAS to Kubernetes API. It may also\nactively send information about cluster events to KAS.\n\n\nWhile agentk is actively communicating with the cluster, KAS represents a\nGitLab server-side component. It is responsible for:\n\n\n- accepting requests from agentk\n\n- authenticating agentk requests by querying GitLab backend\n\n- fetching the agent's configuration from a corresponding Git repository\nusing Gitaly\n\n- polling manifest repositories for GitOps support\n\n\nWe implemented the agent access rights feature to provide access from the\nGitLab frontend to the cluster in a secure and reliable way. To enable the\nfeature, the user should update the agent's configuration file by adding the\n[user_access](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html)\nsection with the following parameters: `projects`, `groups`, and `access_as`\nto specify which projects can access cluster information via the agent and\nhow it should authenticate.\n\n\nOnce this is done, the frontend can connect to the cluster by sending a\nrequest to the Rails controller, which should set a `gitlab_kas cookie`.\nThis cookie is then added to the request sent to KAS together with the agent\nID and Cross-Site Request Forgery (CSRF) token. Upon receiving the request,\nKAS checks the user's authorization and forwards it to agentk, which makes\nan actual request to the Kubernetes API. Then the response goes all the way\nback from the agentk to KAS and finally to the GitLab client.\n\n\n![Kubernetes overview - how it\nworks](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099055229.png)\n\n\nTo integrate this logic on the GitLab frontend and use it within the Vue\napp, we developed a JavaScript library:\n[@gitlab/cluster-client](https://gitlab.com/gitlab-org/cluster-integration/javascript-client).\nIt is generated from the Kubernetes OpenAPI specification using the\ntypescript-fetch generator. It provides all the Kubernetes APIs in a way\nthat can be used in a web browser, along with our `WebSocketWatchManager`\nclass that handles the Watch Aggregator API for efficient real-time updates.\n\n\n## Introducing the Watch API\n\n\nThe most challenging task is to provide **real-time updates** for the\nKubernetes dashboard. Kubernetes introduces the concept of watches as an\nextension of GET requests, exposing the body contents as a [readable\nstream](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams).\nOnce connected to the stream, the Kubernetes API pushes cluster state\nupdates similarly to how the `kubectl get \u003Cresource> --watch` command works.\nThe watch mechanism allows a client to fetch the current state of the\nresource (or resources list) and then subscribe to subsequent changes,\nwithout missing any events. Each event contains a type of modification (one\nof three types: added, modified, or deleted) and the affected object.\n\n\nWithin the `WatchApi` class of the `@gitlab/cluster-client` library, we've\ndeveloped a systematic approach for interacting with the Kubernetes API.\nThis involves fetching a continuous stream of data, processing it line by\nline, and managing events based on their types. Let's explore the key\ncomponents and functionalities of this approach:\n\n\n1. Extending the Kubernetes API: Within the `WatchApi` class, we extend the\nbase Kubernetes API functionality to fetch a continuous stream of data with\na specified path and query parameters. This extension enables efficient\nhandling of large datasets, as the stream is processed line by line.\n2. Decoding and event categorization: Upon receiving the stream, each line,\ntypically representing a JSON object, is decoded. This process extracts relevant\ninformation and categorizes events based on their types.\n3. Internal data management: The `WatchApi` class maintains an internal data\narray to represent the current state of the streamed data, updating it\naccordingly as new data arrives or changes occur. \n\n4. The `WatchApi` class implements methods for registering event listeners,\nsuch as `onData`, `onError`, `onTimeout`, and `onTerminate`. These methods\nallow developers to customize their application's response to events like\ndata updates, errors, and timeouts. \n\n\nThe code also handles scenarios such as invalid content types, timeouts, and\nerrors from the server, emitting corresponding events for clients to handle\nappropriately. **With this straightforward, event-driven approach, the\n`WatchApi` class allows developers to create responsive real-time\napplications efficiently.**\n\n\n![Kubernetes overview - flow\nchart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099055231.png)\n\n## Evolution to WebSocket Aggregation\n\nBuilding on this Watch API foundation, we've introduced the `WebSocketWatchManager`\nto address the limitations of readable streams. This enhanced approach uses the\n`gitlab-agent-watch-api` WebSocket subprotocol to aggregate multiple resource\nwatches within a single persistent connection, bringing several improvements:\n\n- **Multiple watches through one connection:** Instead of opening separate HTTP\nstreams for each Kubernetes resource, they all share a single WebSocket connection.\nThis significantly reduces the connection overhead on both client and server,\nespecially important when monitoring numerous resources across different namespaces.\n\n- **Dynamic watch management:** The protocol supports starting and stopping\nindividual watches on demand through `watch` and `unwatch` messages. The `unwatch`\nfunctionality isn't yet implemented in the JavaScript client, but once added,\nit will stop unnecessary watches when users navigate away from resource views.\n\n- **More reliable connections:** WebSocket connections maintain stable, long-lived\nsessions without the unpredictable drops we experienced with the Watch API's\nHTTP-based streaming. These streaming connections would sometimes terminate unexpectedly,\nrequiring reconnection logic and potentially missing events during the reconnection window.\n\n\nWhen establishing a connection, the client must first request an authentication token\nbecause WebSocket connections cannot send custom headers (like CSRF tokens or cookies)\nafter the initial handshake. This token, obtained via a standard HTTP request with full\nauthentication headers, is then embedded in the WebSocket subprotocol header during\nconnection establishment. Once connected to the `/watch` endpoint, the client starts\nindividual watches by sending messages with resource parameters, and the server\nstreams back watch events as Kubernetes resources change.\n\n\nThe system falls back to the original `WatchApi` implementation when WebSocket\nconnections fail for any reason — whether due to network infrastructure limitations,\nconnection failures, or authentication issues. This dual approach delivers WebSocket\nperformance benefits where possible, while maintaining full functionality through the\nWatch API's streaming method when needed. The switch happens automatically, delivering\nconsistent real-time functionality without requiring user awareness or intervention.\n\n\n## How is the Kubernetes overview integrated with the GitLab frontend?\n\n\nCurrently, we have two Kubernetes integrations within the product: the\nKubernetes overview section for the Environments and the full Kubernetes\ndashboard as a separate view. The latter is a major effort of representing\nall the available Kubernetes resources with filtering and sorting\ncapabilities and a detailed view with the full information on the metadata,\nspec, and status of the resource. This initiative is now on hold while we\nare searching for the most useful ways of representing the Kubernetes\nresources related to an environment.\n\n\n[The Kubernetes\noverview](https://docs.gitlab.com/ee/ci/environments/kubernetes_dashboard.html)\non the Environments page is a detailed view of the Kubernetes resources\nrelated to a specific environment. To access the cluster state view, the\nuser should select an agent installed in the cluster with the appropriate\naccess rights, provide a namespace (optionally), and select a related Flux\nresource.\n\n\nThe view renders a list of Kubernetes pods and services filtered by the\nnamespace representing their statuses as well as the Flux sync status.\nClicking each resource opens a detailed view with more information for easy\nissue spotting and high-level debugging. \n\n\n![Kubernetes overview - list of Kubernetes pods and\nservices](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099055233.png)\n\n\nWe need to set up a correct configuration object that will be used for all\nthe API requests. In the configuration, we need to specify the URL provided\nby the KAS, that proxies the Kubernetes APIs; the GitLab agent ID to connect\nwith; and the CSRF token. We need to include cookies so that the\n`kas_cookie` gets picked up and sent within the request.\n\n\n```javascript\n\ncreateK8sAccessConfig({ kasTunnelUrl, gitlabAgentId }) {\n  return {\n    basePath: kasTunnelUrl,\n    headers: {\n      'GitLab-Agent-Id': gitlabAgentId,\n      ...csrf.headers,\n    },\n    credentials: 'include',\n  };\n}\n\n```\n\nThis configuration object serves as the foundation for all Kubernetes API interactions\nthrough KAS, whether using standard API calls, Watch API streaming, or WebSocket\nconnections. Each connection method builds upon this base configuration with its specific\nrequirements.\n\n\nAll the API requests are implemented as GraphQL client queries for\nefficiency, flexibility, and ease of development. The query structure\nenables clients to fetch data from various sources in one request. With\nclear schema definitions, GraphQL minimizes errors and enhances developer\nefficiency. The WebSocket implementation complements this by managing\nreal-time updates through a single persistent connection, reducing\nthe need for multiple parallel streaming connections while maintaining\nthe same GraphQL query structure for data updates.\n\n\nWhen first rendering the Kubernetes overview, the frontend requests static\nlists of pods, services, and Flux resource (either HelmRelease or\nKustomization). The fetch request is needed to render the empty view\ncorrectly. If the frontend tried to subscribe to the Watch API stream and\none of the resource lists was empty, we would wait for the updates forever\nand never show the actual result — 0 resources. In the case of pods and\nservices, after the initial request, we subscribe to the stream even if an\nempty list was received to reflect any cluster state changes. For the Flux\nresource, the changes that the user would expect the resource to appear\nafter the initial request are low. We use the empty response here as an\nopportunity to provide more information about the feature and its setup. \n\n\n![Kubernetes overview - flux sync status\nunavailable](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099055235.png)\n\n\nAfter rendering the initial result, the frontend establishes real-time\nconnections to monitor changes. The implementation attempts the WebSocket\nconnection first, falling back to the Watch API streaming when unavailable.\nWith WebSocket connections, a single connection handles all resource watches,\nwhich maintains internal streams for each resource and updates them based on\nincoming events (`ADDED`, `MODIFIED`, `DELETED`). In Watch API fallback mode,\nthe frontend makes requests with the `?watch=true` query parameter, creating\nseparate watchers for each event type:\n\n\n```javascript\n\nwatcher.on(EVENT_DATA, (data) => {\n  result = data.map(mapWorkloadItem);\n  client.writeQuery({\n    query,\n    variables: { configuration, namespace },\n    data: { [queryField]: result },\n  });\n\n  updateConnectionStatus(client, {\n    configuration,\n    namespace,\n    resourceType: queryField,\n    status: connectionStatus.connected,\n  });\n});\n\n```\n\nBoth approaches follow the same data processing pattern:\n\n- Transform the data to ensure a consistent structure\n\n- Update the Apollo cache to trigger UI updates\n\n- Run a mutation to update the connection status indicator\n\n\nAs we show the detailed information for each resource, we rely on having the\nstatus, spec, and metadata fields with the annotations and labels included.\nThe Kubernetes API wouldn't always send this information, which could break\nthe UI and throw errors from the GraphQL client. We transform the received\ndata first to avoid these issues. We also add the `__typename` so that we\ncan better define the data types and simplify the queries by reusing the\nshared fragments.\n\n\nAfter data stabilization, we update the Apollo cache so that the frontend\nre-renders the views accordingly to reflect cluster state changes.\nInterestingly, we can visualize exactly what happens in the cluster — for\nexample, when deleting the pods, Kubernetes first creates the new ones in\nthe pending state, and only then removes the old pods. Thus, for a moment, we\ncan see double the amount of pods. We can also verify how the pods proceed\nfrom one state to another in real-time. This is done with the combination of\nadded, deleted, and modified events received from the Kubernetes APIs and\nprocessed by either the `WebSocketWatchManager` or the` WatchApi` class of\nthe `@gitlab/cluster-client` library.\n\n\n![Kubernetes overview - states of connection\nstatus](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099055/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099055236.gif)\n\n\nThe connection status indicator remains important for both connection types,\nthough for different reasons. With WebSocket connections, while they're more\nstable and long-lived, users still need to know if the connection drops due\nto network issues or authentication expiry. With Watch API streaming, the\nconnections have timeout limitations and are more prone to disconnection.\nTo achieve this visibility, we introduced a `k8sConnection` query together with\nthe `reconnectToCluster` mutation. The UI displays a badge with a tooltip showing\nthree connection states:\n\n- **Connecting:** Set when initiating either the connection\n\n- **Connected:** Updated when the first data arrives through either connection type\n\n- **Disconnected:** Triggered when WebSocket connections fail, Watch API streams timeout, or any connection errors occur\n\n\nWhen disconnection occurs, users can click to reconnect without refreshing the browser.\nRelying on the user action to reconnect to the stream helps us save resources and only\nrequest the necessary data while ensuring the accurate cluster state is available for\nthe user at any time.\n\n\n## What's next?\n\n\nLeveraging the Kubernetes built-in functionality for watching the Readable\nstream helped us to build the functionality quickly and provide the\nKubernetes UI solution to our customers, getting early feedback and\nadjusting the product direction. The initial Watch API approach, while\npresenting challenges like connection timeouts and reconnection needs,\ngave us valuable insights into real-world usage patterns. The WebSocket\nimplementation has addressed many of these initial challenges, providing\nmore stable connections and better resource efficiency through multiplexed watches.\nBoth the Watch API and WebSocket approaches continue to evolve based on user needs.\n\n\nThere are opportunities for enhancement that could improve the user experience further.\nThe WebSocket protocol already supports `unwatch` messages for dynamic watch management,\nthough this isn't yet implemented in the JavaScript client. Fuller error handling and\nsupport for additional Kubernetes resource types could also expand the dashboard's capabilities.\n\n\nIf you're interested in contributing to the Kubernetes dashboard functionality,\nexploring enhancement opportunities, or sharing your use cases, join the\ndiscussion in our [Kubernetes Dashboard epic](https://gitlab.com/groups/gitlab-org/-/epics/2493).\n\n\n> Curious to learn more or want to try out this functionality? Visit our\n[Kubernetes Dashboard\ndocumentation](https://docs.gitlab.com/ee/ci/environments/kubernetes_dashboard.html)\nfor more details and configuration tips.\n",[1241,696,9],{"slug":3190,"featured":90,"template":700},"kubernetes-overview-operate-cluster-data-on-the-frontend","content:en-us:blog:kubernetes-overview-operate-cluster-data-on-the-frontend.yml","Kubernetes Overview Operate Cluster Data On The Frontend","en-us/blog/kubernetes-overview-operate-cluster-data-on-the-frontend.yml","en-us/blog/kubernetes-overview-operate-cluster-data-on-the-frontend",{"_path":3196,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3197,"content":3203,"config":3208,"_id":3210,"_type":13,"title":3211,"_source":15,"_file":3212,"_stem":3213,"_extension":18},"/en-us/blog/learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions",{"title":3198,"description":3199,"ogTitle":3198,"ogDescription":3199,"noIndex":6,"ogImage":3200,"ogUrl":3201,"ogSiteName":685,"ogType":686,"canonicalUrls":3201,"schema":3202},"Learn advanced Rust programming with a little help from AI","Use this guided tutorial, along with AI-powered GitLab Duo Code Suggestions, to continue learning advanced Rust programming.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662439/Blog/Hero%20Images/codewithheart.png","https://about.gitlab.com/blog/learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learn advanced Rust programming with a little help from AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2023-10-12\",\n      }",{"title":3198,"description":3199,"authors":3204,"heroImage":3200,"date":3205,"body":3206,"category":718,"tags":3207},[1646],"2023-10-12","When I started learning a new programming language more than 20 years ago,\nwe had access to the Visual Studio 6 MSDN library, installed from 6 CD-ROMs.\nAlgorithms with pen and paper, design pattern books, and MSDN queries to\nfigure out the correct type were often time-consuming. Learning a new\nprogramming language changed fundamentally in the era of remote\ncollaboration and artificial intelligence (AI). Now you can spin up a\n[remote development\nworkspace](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/),\nshare your screen, and engage in a group programming session. With the help\nof [GitLab Duo Code Suggestions](/gitlab-duo/), you always have an\nintelligent partner at your fingertips. Code Suggestions can learn from your\nprogramming style and experience. They only need input and context to\nprovide you with the most efficient suggestions.\n\n\nIn this tutorial, we build on the [getting started blog\npost](/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/)\nand design and create a simple feed reader application.\n\n\n- [Preparations](#preparations)\n    - [Code Suggestions](#code-suggestions)\n- [Continue learning Rust](#continue-learning-rust)\n    - [Hello, Reader App](#hello-reader-app)\n    - [Initialize project](#initialize-project)\n    - [Define RSS feed URLs](#define-rss-feed-urls)\n- [Modules](#modules)\n    - [Call the module function in main()](#call-the-module-function-in-main)\n- [Crates](#crates)\n    - [feed-rs: parse XML feed](#feed-rs-parse-xml-feed)\n- [Runtime configuration: Program\narguments](#runtime-configuration-program-arguments)\n    - [User input error handling](#user-input-error-handling)\n- [Persistence and data storage](#persistence-and-data-storage)\n\n- [Optimization](#optimization)\n    - [Asynchronous execution](#asynchronous-execution)\n    - [Spawning threads](#spawning-threads)\n    - [Function scopes, threads, and closures](#function-scopes-threads-and-closures)\n- [Parse feed XML into objects](#parse-feed-xml-into-object-types)\n    - [Map generic feed data types](#map-generic-feed-data-types)\n    - [Error handling with Option::unwrap()](#error-handling-with-option-unwrap)\n- [Benchmarks](#benchmarks)\n    - [Sequential vs. Parallel execution benchmark](#sequential-vs-parallel-execution-benchmark)\n    - [CI/CD with Rust caching](#cicd-with-rust-caching)\n- [What is next](#what-is-next)\n    - [Async learning exercises](#async-learning-exercises)\n    - [Share your feedback](#share-your-feedback)\n\n## Preparations\n\nBefore diving into the source code, make sure to set up [VS\nCode](/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/#vs-code)\nand [your development environment with\nRust](/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/#development-environment-for-rust).\n\n\n### Code Suggestions\n\nFamiliarize yourself with suggestions before actually verifying the\nsuggestions. GitLab Duo Code Suggestions are provided as you type, so you do\nnot need use specific keyboard shortcuts. To accept a code suggestion, press\nthe `tab` key. Also note that writing new code works more reliably than\nrefactoring existing code. AI is non-deterministic, which means that the\nsame suggestion may not be repeated after deleting the code suggestion.\nWhile Code Suggestions is in Beta, we are working on improving the accuracy\nof generated content overall. Please review the [known\nlimitations](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#known-limitations),\nas this could affect your learning experience.\n\n\n**Tip:** The latest release of Code Suggestions supports multi-line\ninstructions. You can refine the specifications to your needs to get better\nsuggestions.\n\n\n```rust\n    // Create a function that iterates over the source array\n    // and fetches the data using HTTP from the RSS feed items.\n    // Store the results in a new hash map.\n    // Print the hash map to the terminal.\n```\n\n\nThe VS Code extension overlay is shown when offering a suggestion. You can\nuse the `tab` key to accept the suggested line(s), or `cmd cursor right` to\naccept one word. Additionally, the three dots menu allows you to always show\nthe toolbar.\n\n\n![VS Code GitLab Duo Code Suggestions overlay with\ninstructions](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_code_suggestions_options_overlay_keep_toolbar.png){:\n.shadow}\n\n\n## Continue learning Rust\n\nNow, let us continue learning Rust, which is one of the [supported languages\nin Code\nSuggestions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#supported-languages).\n[Rust by Example](https://doc.rust-lang.org/rust-by-example/) provides an\nexcellent tutorial for beginners, together with the official [Rust\nbook](https://doc.rust-lang.org/book/). Both resources are referenced\nthroughout this blog post.\n\n\n### Hello, Reader App\n\nThere are many ways to create an application and learn Rust. Some of them\ninvolve using existing Rust libraries - so-called `Crates`. We will use them\na bit further into the blog post. For example, you could create a\ncommand-line app that processes images and writes the result to a file.\nSolving a classic maze or writing a Sudoku solver can also be a fun\nchallenge. Game development is another option. The book [Hands-on\nRust](https://hands-on-rust.com/) provides a thorough learning path by\ncreating a dungeon crawler game. My colleague Fatima Sarah Khalid started\nthe [Dragon Realm in C++ with a little help from\nAI](/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions/)\n-- check it out, too.\n\n\nHere is a real use case that helps solve an actual problem: Collecting\nimportant information from different sources into RSS feeds for (security)\nreleases, blog posts, and social discussion forums like Hacker News. Often,\nwe want to filter for specific keywords or versions mentioned in the\nupdates. These requirements allow us to formulate a requirements list for\nour application:\n\n\n1. Fetch data from different sources (HTTP websites, REST API, RSS feeds).\nRSS feeds in the first iteration.\n\n1. Parse the data.\n\n1. Present the data to the user, or write it to disk.\n\n1. Optimize performance.\n\n\nThe following example application output will be available after the\nlearning steps in this blog post:\n\n\n![VS Code Terminal, cargo run with formatted feed entries\noutput](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_cargo_run_formatted_output_final.png)\n\n\nThe application should be modular and build the foundation to add more data\ntypes, filters, and hooks to trigger actions at a later point.\n\n\n### Initialize project\n\nReminder: `cargo init` in the project root creates the file structure,\nincluding the `main()` entrypoint. Therefore, we will learn how to create\nand use Rust modules in the next step.\n\n\nCreate a new directory called `learn-rust-ai-app-reader`, change into it and\nrun `cargo init`. This command implicitly runs `git init` to initialize a\nnew Git repository locally. The remaining step is to configure the Git\nremote repository path, for example,\n`https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai-app-reader`.\nPlease adjust the path for your namespace. Pushing the Git repository\n[automatically creates a new private project in\nGitLab](https://docs.gitlab.com/ee/user/project/#create-a-new-project-with-git-push).\n\n\n```shell\n\nmkdir learn-rust-ai-app-reader\n\ncd learn-rust-ai-app-reader\n\n\ncargo init\n\n\ngit remote add origin\nhttps://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai-app-reader.git\n\ngit push --set-upstream origin main\n\n```\n\n\nOpen VS Code from the newly created directory. The `code` CLI will spawn a\nnew VS Code window on macOS.\n\n\n```shell\n\ncode .\n\n```\n\n\n### Define RSS feed URLs\n\nAdd a new hashmap to store the RSS feed URLs inside the `src/main.rs` file\nin the `main()` function. You can instruct GitLab Duo Code Suggestions with\na multi-line comment to create a\n[`HashMap`](https://doc.rust-lang.org/stable/std/collections/struct.HashMap.html)\nobject, and initialize it with default values for Hacker News, and\nTechCrunch. Note: Verify that the URLs are correct when you get suggestions.\n\n\n```rust\n\nfn main() {\n    // Define RSS feed URLs in the variable rss_feeds\n    // Use a HashMap\n    // Add Hacker News and TechCrunch\n    // Ensure to use String as type\n\n}\n\n```\n\n\nNote that the code comment provides instructions for:\n\n\n1. The variable name `rss_feeds`.\n\n2. The `HashMap` type.\n\n3. Initial seed key/value pairs.\n\n4. String as type (can be seen with `to_string()` calls).\n\n\nOne possible suggested path can be as follows:\n\n\n```rust\n\nuse std::collections::HashMap;\n\n\nfn main() {\n    // Define RSS feed URLs in the variable rss_feeds\n    // Use a HashMap\n    // Add Hacker News and TechCrunch\n    // Ensure to use String as type\n    let rss_feeds = HashMap::from([\n        (\"Hacker News\".to_string(), \"https://news.ycombinator.com/rss\".to_string()),\n        (\"TechCrunch\".to_string(), \"https://techcrunch.com/feed/\".to_string()),\n    ]);\n\n}\n\n```\n\n\n![VS Code with Code Suggestions for RSS feed URLs for Hacker News and\nTechCrunch](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_main_array_rss_feed_urls_suggested.png)\n\n\nOpen a new terminal in VS Code (cmd shift p - search for `terminal`), and\nrun `cargo build` to build the changes. The error message instructs you to\nadd the `use std::collections::HashMap;` import.\n\n\nThe next step is to do something with the RSS feed URLs. [The previous blog\npost](/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/)\ntaught us to split code into functions. We want to organize the code more\nmodularly for our reader application, and use Rust modules.\n\n\n## Modules\n\n[Modules](https://doc.rust-lang.org/rust-by-example/mod.html) help with\norganizing code. They can also be used to hide functions into the module\nscope, limiting access to them from the main() scope. In our reader\napplication, we want to fetch the RSS feed content, and parse the XML\nresponse. The `main()` caller should only be able to access the\n`get_feeds()` function, while other functionality is only available in the\nmodule.\n\n\nCreate a new file `feed_reader.rs` in the `src/` directory. Instruct Code\nSuggestions to create a public module named `feed_reader`, and a public\nfunction `get_feeds()` with a String HashMap as input. Important: The file\nand module names need to be the same, following the [Rust module\nstructure](https://doc.rust-lang.org/book/ch07-02-defining-modules-to-control-scope-and-privacy.html).\n\n\n![Code Suggestions: Create public module, with function and input\ntypes](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_public_module_function_input.png){:\n.shadow}\n\n\nInstructing Code Suggestions with the input variable name and type will also\nimport the required `std::collections::HashMap` module. Tip: Experiment with\nthe comments, and refine the variable types to land the best results.\nPassing function parameters as object references is considered best practice\nin Rust, for example.\n\n\n```rust\n\n// Create public module feed_reader\n\n// Define get_feeds() function which takes rss_feeds as String HashMap\nreference as input\n\npub mod feed_reader {\n    use std::collections::HashMap;\n\n    pub fn get_feeds(rss_feeds: &HashMap\u003CString, String>) {\n        // Do something with the RSS feeds\n    }\n}\n\n```\n\n\n![Code Suggestions: Public module with `get_feeds()` function, and suggested\ninput\nvariable](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_public_module_function_input.png){:\n.shadow}\n\n\nInside the function, continue to instruct Code Suggestions with the\nfollowing steps:\n\n\n1. `// Iterate over the RSS feed URLs`\n\n2. `// Fetch URL content`\n\n3. `// Parse XML body`\n\n4. `// Print the result`\n\n\n![Code Suggestions: Public module with `get_feeds()` function, step 1:\nIterate](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_module_function_01_iterate.png){:\n.shadow}\n\n\n![Code Suggestions: Public module with `get_feeds()` function, step 2: Fetch\nURL\ncontent](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_module_function_02_fetch_content.png){:\n.shadow}\n\n\n![Code Suggestions: Public module with `get_feeds()` function, step 3: Parse\nXML\nbody](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_module_function_03_parse_body.png){:\n.shadow}\n\n\n![Code Suggestions: Public module with `get_feeds()` function, step 4: Print\nthe\nresults](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_module_function_04_print_result.png){:\n.shadow}\n\n\nThe following code can be suggested:\n\n\n```rust\n\n// Create public module feed_reader\n\n// Define get_feeds() function which takes rss_feeds as String HashMap\nreference as input\n\npub mod feed_reader {\n    use std::collections::HashMap;\n\n    pub fn get_feeds(rss_feeds: &HashMap\u003CString, String>) {\n        // Iterate over the RSS feed URLs\n        for (name, url) in rss_feeds {\n            println!(\"{}: {}\", name, url);\n\n            // Fetch URL content\n            let body = reqwest::blocking::get(url).unwrap().text().unwrap();\n\n            // Parse XML body\n            let parsed_body = roxmltree::Document::parse(&body).unwrap();\n\n            // Print the result\n            println!(\"{:#?}\", parsed_body);\n        }\n    }\n}\n\n```\n\n\nYou see a new keyword here:\n[`unwrap()`](https://doc.rust-lang.org/rust-by-example/error/option_unwrap.html).\nRust does not support `null` values, and uses the [`Option`\ntype](https://doc.rust-lang.org/rust-by-example/std/option.html) for any\nvalue. If you are certain to use a specific wrapped type, for example,\n`Text` or `String`, you can call the `unwrap()` method to get the value. The\n`unwrap()` method will panic if the value is `None`.\n\n\n**Note** Code Suggestions referred to the `reqwest::blocking::get` function\nfor the `// Fetch URL content` comment instruction. The [`reqwest`\ncrate](https://docs.rs/reqwest/latest/reqwest/) name is intentional and not\na typo. It provides a convenient, higher-level HTTP client for async and\nblocking requests.\n\n\nParsing the XML body is tricky - you might get different results, and the\nschema is not the same for every RSS feed URL. Let us try to call the\n`get_feeds()` function, and then work on improving the code.\n\n\n### Call the module function in main()\n\n\nThe main() function does not know about the `get_feeds()` function yet, so\nwe need to import its module. In other programming languages, you might have\nseen the keywords `include` or `import`. The Rust module system is\ndifferent.\n\n\nModules are organized in path directories. In our example, both source files\nexist on the same directory level. `feed_reader.rs` is interpreted as crate,\ncontaining one module called `feed_reader`, which defines the function\n`get_feeds()`.\n\n\n```\n\nsrc/\n  main.rs\n  feed_reader.rs\n```\n\n\nIn order to access `get_feeds()` from the `feed_reader.rs` file, we need to\n[bring module\npath](https://doc.rust-lang.org/book/ch07-04-bringing-paths-into-scope-with-the-use-keyword.html)\ninto the `main.rs` scope first, and then call the full function path.\n\n\n```rust\n\nmod feed_reader;\n\n\nfn main() {\n\n    feed_reader::feed_reader::get_feeds(&rss_feeds);\n\n```\n\n\nAlternatively, we can import the full function path with the `use` keyword,\nand later use the short function name.\n\n\n```rust\n\nmod feed_reader;\n\nuse feed_reader::feed_reader::get_feeds;\n\n\nfn main() {\n\n    get_feeds(&rss_feeds);\n\n```\n\n\n**Tip:** I highly recommend reading the [Clear explanation of the Rust\nmodule system blog\npost](https://www.sheshbabu.com/posts/rust-module-system/) to get a better\nvisual understanding.\n\n\n```diff\n\n\nfn main() {\n    // ...\n\n    // Print feed_reader get_feeds() output\n    println!(\"{}\", feed_reader::get_feeds(&rss_feeds));\n```\n\n\n```rust\n\nuse std::collections::HashMap;\n\n\nmod feed_reader;\n\n// Alternative: Import full function path\n\n//use feed_reader::feed_reader::get_feeds;\n\n\nfn main() {\n    // Define RSS feed URLs in the variable rss_feeds\n    // Use a HashMap\n    // Add Hacker News and TechCrunch\n    // Ensure to use String as type\n    let rss_feeds = HashMap::from([\n        (\"Hacker News\".to_string(), \"https://news.ycombinator.com/rss\".to_string()),\n        (\"TechCrunch\".to_string(), \"https://techcrunch.com/feed/\".to_string()),\n    ]);\n\n    // Call get_feeds() from feed_reader module\n    feed_reader::feed_reader::get_feeds(&rss_feeds);\n    // Alternative: Imported full path, use short path here.\n    //get_feeds(&rss_feeds);\n}\n\n```\n\n\nRun `cargo build` in the terminal again to build the code.\n\n\n```shell\n\ncargo build\n\n```\n\n\nPotential build errors when Code Suggestions refer to common code and\nlibraries for HTTP requests, and XML parsing:\n\n\n1. Error: `could not find blocking in reqwest`. Solution: Enable the\n`blocking` feature for the crate in `Config.toml`: `reqwest = { version =\n\"0.11.20\", features = [\"blocking\"] }`.\n\n2. Error: `failed to resolve: use of undeclared crate or module reqwest`.\nSolution: Add the `reqwest` crate.\n\n3. Error: `failed to resolve: use of undeclared crate or module roxmltree`.\nSolution: Add the `roxmltree` crate.\n\n\n```shell\n\nvim Config.toml\n\n\nreqwest = { version = \"0.11.20\", features = [\"blocking\"] }\n\n```\n\n\n```shell\n\ncargo add reqwest\n\ncargo add roxmltree\n\n```\n\n\n**Tip:** Copy the error message string, with a leading `Rust \u003Cerror\nmessage>` into your preferred browser to check whether a missing crate is\navailable. Usually this search leads to a result on crates.io and you can\nadd the missing dependencies.\n\n\nWhen the build is successful, run the code with `cargo run` and inspect the\nHacker News RSS feed output.\n\n\n![VS Code terminal, cargo run to fetch Hacker News XML\nfeed](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_fetch_rss_feed_output_hacker_news.png){:\n.shadow}\n\n\nWhat is next with parsing the XML body into human-readable format? In the\nnext section, we will learn about existing solutions and how Rust crates\ncome into play.\n\n\n## Crates\n\nRSS feeds share a common set of protocols and specifications. It feels like\nreinventing the wheel to parse XML items and understand the lower object\nstructure. Recommendation for these types of tasks: Look whether someone\nelse had the same problem already and might have created code to solve the\nproblem.\n\n\nReusable library code in Rust is organized in so-called\n[`Crates`](https://doc.rust-lang.org/rust-by-example/crates.html), and made\navailable in packages, and the package registry on crates.io. You can add\nthese dependencies to your project by editing the `Config.toml` in the\n`[dependencies]` section, or using `cargo add \u003Cname>`.\n\n\nFor the reader app, we want to use the [feed-rs\ncrate](https://crates.io/crates/feed-rs). Open a new terminal, and run the\nfollowing command:\n\n\n```shell\n\ncargo add feed-rs\n\n```\n\n\n![VS Code Terminal Terminal: Add crate, verify in\nConfig.toml](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_rust_crate_add_feed-rs_explained.png)\n\n\n### feed-rs: parse XML feed\n\nNavigate into `src/feed_reader.rs` and modify the part where we parse the\nXML body. Code Suggestions understands how to call the `feed-rs` crate\n`parser::parse` function -- there is only one specialty here: `feed-rs`\n[expects string input as raw\nbytes](https://docs.rs/feed-rs/latest/feed_rs/parser/fn.parse_with_uri.html)\nto determine the encoding itself. We can provide instructions in the comment\nto get the expected result though.\n\n\n```rust\n            // Parse XML body with feed_rs parser, input in bytes\n            let parsed_body = feed_rs::parser::parse(body.as_bytes()).unwrap();\n```\n\n\n![Code Suggestions: Public module with `get_feeds()` function, step 5:\nModify XML parser to\nfeed-rs](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_module_function_05_use_feed_rs_to_parse.png){:\n.shadow}\n\n\nThe benefit of using `feed-rs` is not immediately visible until you see the\nprinted output with `cargo run`: All keys and values are mapped to their\nrespective Rust object types, and can be used for further operations.\n\n\n![VS Code terminal, cargo run to fetch Hacker News XML\nfeed](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_fetch_rss_feed_output_hacker_news_feed_rs.png){:\n.shadow}\n\n\n## Runtime configuration: Program arguments\n\nUntil now, we have run the program with hard-coded RSS feed values compiled\ninto the binary. The next step is allowing to configure the RSS feeds at\nruntime.\n\n\nRust provides [program\narguments](https://doc.rust-lang.org/rust-by-example/std_misc/arg.html) in\nthe standard misc library. [Parsing the\narguments](https://doc.rust-lang.org/rust-by-example/std_misc/arg/matching.html)\nprovides a better and faster learning experience than aiming for advanced\nprogram argument parsers (for example, the\n[clap](https://docs.rs/clap/latest/clap/) crate), or moving the program\nparameters into a configuration file and format\n([TOML](https://toml.io/en/), YAML). You are reading these lines after I\ntried and failed with different routes for the best learning experience.\nThis should not stop you from taking the challenge to configure RSS feeds in\nalternative ways.\n\n\nAs a boring solution, the command parameters can be passed as `\"name,url\"`\nstring value pairs, and then are split by the `,` character to extract the\nname and URL values. The comment instructs Code Suggestions to perform these\noperations and extend the `rss_feeds` HashMap with the new values. Note that\nthe variable might not be mutable, and, therefore, needs to be modified to\n`let mut rss_feeds`.\n\n\nNavigate into `src/main.rs` and add the following code to the `main()`\nfunction after the `rss_feeds` variable. Start with a comment to define the\nprogram arguments, and check the suggested code snippets.\n\n\n```rust\n    // Program args, format \"name,url\"\n    // Split value by , into name, url and add to rss_feeds\n```\n\n\n![Code suggestions for program arguments, and splitting name,URL values for\nthe rss_feeds\nvariable](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_program_args_boring_solution.png){:\n.shadow}\n\n\nThe full code example can look like the following:\n\n\n```rust\n\nfn main() {\n    // Define RSS feed URLs in the variable rss_feeds\n    // Use a HashMap\n    // Add Hacker News and TechCrunch\n    // Ensure to use String as type\n    let mut rss_feeds = HashMap::from([\n        (\"Hacker News\".to_string(), \"https://news.ycombinator.com/rss\".to_string()),\n        (\"TechCrunch\".to_string(), \"https://techcrunch.com/feed/\".to_string()),\n    ]);\n\n    // Program args, format \"name,url\"\n    // Split value by , into name, url and add to rss_feeds\n    for arg in std::env::args().skip(1) {\n        let mut split = arg.split(\",\");\n        let name = split.next().unwrap();\n        let url = split.next().unwrap();\n        rss_feeds.insert(name.to_string(), url.to_string());\n    }\n\n    // Call get_feeds() from feed_reader module\n    feed_reader::feed_reader::get_feeds(&rss_feeds);\n    // Alternative: Imported full path, use short path here.\n    //get_feeds(&rss_feeds);\n}\n\n```\n\n\nYou can pass program arguments directly to the `cargo run` command,\npreceding the arguments with `--`. Enclose all arguments with double quotes,\nput the name followed by a comma and the RSS feed URL as argument. Separate\nall arguments with whitespaces.\n\n\n```\n\ncargo build\n\n\ncargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n```\n\n\n![VS Code terminal, RSS feed output example for the GitLab\nblog](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_gitlab_blog_rss_feed_example.png){:\n.shadow}\n\n\n### User input error handling\n\nIf the provided user input does not match the program expectation, we need\nto [throw an error](https://doc.rust-lang.org/rust-by-example/error.html)\nand help the caller to fix the program arguments. For example, passing a\nmalformed URL format should be treated as a runtime error. Instruct Code\nSuggestions with a code comment to throw an error if the URL is not valid.\n\n\n```rust\n    // Ensure that URL contains a valid format, otherwise throw an error\n```\n\n\nOne possible solution is to check if the `url` variable starts with\n`http://` or `https://`. If not, throw an error using the [panic!\nmacro](https://doc.rust-lang.org/rust-by-example/std/panic.html). The full\ncode example looks like the following:\n\n\n```rust\n    // Program args, format \"name,url\"\n    // Split value by , into name, url and add to rss_feeds\n    for arg in std::env::args().skip(1) {\n        let mut split = arg.split(\",\");\n        let name = split.next().unwrap();\n        let url = split.next().unwrap();\n\n        // Ensure that URL contains a valid format, otherwise throw an error\n        if !url.starts_with(\"http://\") && !url.starts_with(\"https://\") {\n            panic!(\"Invalid URL format: {}\", url);\n        }\n\n        rss_feeds.insert(name.to_string(), url.to_string());\n    }\n```\n\n\nTest the error handling with removing a `:` in one of the URL strings. Add\nthe `RUST_BACKTRACE=full` environment variable to get more verbose output\nwhen the `panic()` call happens.\n\n\n```\n\nRUST_BACKTRACE=full cargo run -- \"GitLab\nBlog,https://about.gitlab.com/atom.xml\" \"CNCF,https//www.cncf.io/feed/\"\n\n```\n\n\n![VS Code Terminal with wrong URL format, panic error\nbacktrace](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_url_format_error_panic_backtrace.png){:\n.shadow}\n\n\n## Persistence and data storage\n\nThe boring solution for storing the feed data is to dump the parsed body\ninto a new file. Instruct Code Suggestions to use a pattern that includes\nthe RSS feed name, and the current ISO date.\n\n\n```rust\n    // Parse XML body with feed_rs parser, input in bytes\n    let parsed_body = feed_rs::parser::parse(body.as_bytes()).unwrap();\n\n    // Print the result\n    println!(\"{:#?}\", parsed_body);\n\n    // Dump the parsed body to a file, as name-current-iso-date.xml\n    let now = chrono::offset::Local::now();\n    let filename = format!(\"{}-{}.xml\", name, now.format(\"%Y-%m-%d\"));\n    let mut file = std::fs::File::create(filename).unwrap();\n    file.write_all(body.as_bytes()).unwrap();\n```\n\n\nA possible suggestion will include using the [chrono\ncrate](https://crates.io/crates/chrono). Add it using `cargo add chrono` and\nthen invoke `cargo build` and `cargo run` again.\n\n\nThe files are written into the same directory where `cargo run` was\nexecuted. If you are executing the binary direcly in the `target/debug/`\ndirectory, all files will be dumped there.\n\n\n![VS Code with CNCF RSS feed content file, saved on\ndisk](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_cncf_rss_feed_saved_on_disk.png)\n\n\n## Optimization\n\nThe entries in the `rss_feeds` variable are executed sequentially. Imagine\nhaving a list of 100+ URLs configured - this could take a long time to fetch\nand process. What if we could execute multiple fetch requests in parallel?\n\n\n### Asynchronous execution\n\nRust provides [threads](https://doc.rust-lang.org/book/ch16-01-threads.html)\nfor asynchronous execution.\n\n\nThe simplest solution will be spawning a thread for each RSS feed URL. We\nwill discuss optimization strategies later. Before you continue with\nparallel execution, measure the sequential code execution time by preceding\nthe `time` command with `cargo run`.\n\n\n```\n\ntime cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n\n0.21s user 0.08s system 10% cpu 2.898 total\n\n```\n\n\nNote that this exercise could require more manual code work. It is\nrecommended to persist the sequential working state in a new Git commit and\nbranch `sequential-exec`, to better compare the impact of parallel\nexecution.\n\n\n```shell\n\ngit commit -avm \"Sequential execution working\"\n\ngit checkout -b sequential-exec\n\ngit push -u origin sequential-exec\n\n\ngit checkout main\n\n```\n\n\n### Spawning threads\n\nOpen `src/feed_reader.rs` and refactor the `get_feeds()` function. Start\nwith a Git commit for the current state, and then delete the contents of the\nfunction scope. Add the following code comments with instructions for Code\nSuggestions:\n\n\n1. `// Store threads in vector`: Store thread handles in a vector, so we can\nwait for them to finish at the end of the function call.\n\n2. `// Loop over rss_feeds and spawn threads`: Create boilerplate code for\niterating over all RSS feeds, and spawn a new thread.\n\n\nAdd the following `use` statements to work with the `thread` and `time`\nmodules.\n\n\n```rust\n    use std::thread;\n    use std::time::Duration;\n```\n\n\nContinue writing the code, and close the for loop. Code Suggestions will\nautomatically propose adding the thread handle in the `threads` vector\nvariable, and offer to join the threads at the end of the function.\n\n\n```rust\n    pub fn get_feeds(rss_feeds: &HashMap\u003CString, String>) {\n\n        // Store threads in vector\n        let mut threads: Vec\u003Cthread::JoinHandle\u003C()>> = Vec::new();\n\n        // Loop over rss_feeds and spawn threads\n        for (name, url) in rss_feeds {\n            let thread_name = name.clone();\n            let thread_url = url.clone();\n            let thread = thread::spawn(move || {\n\n            });\n            threads.push(thread);\n        }\n\n        // Join threads\n        for thread in threads {\n            thread.join().unwrap();\n        }\n    }\n```\n\n\nAdd the `thread` crate, build and run the code again.\n\n\n```shell\n\ncargo add thread\n\n\ncargo build\n\n\ncargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n```\n\n\nAt this stage, no data is processed or printed. Before we continue re-adding\nthe functionality, let us learn about the newly introduced keywords here.\n\n\n### Function scopes, threads, and closures\n\nThe suggested code brings new keywords and design patterns to learn. The\nthread handle is of the type `thread::JoinHandle`, indicating that we can\nuse it to wait for the threads to finish\n([join()](https://doc.rust-lang.org/book/ch16-01-threads.html#waiting-for-all-threads-to-finish-using-join-handles)).\n\n\n`thread::spawn()` spawns a new thread, where we can pass a function object.\nIn this case, a\n[closure](https://doc.rust-lang.org/book/ch13-01-closures.html) expression\nis passed as anonymous function. Closure inputs are passed using the `||`\nsyntax. You will recognize the [`move`\nClosure](https://doc.rust-lang.org/book/ch16-01-threads.html#using-move-closures-with-threads),\nwhich moves the function scoped variables into the thread scope. This avoids\nmanually specifying which variables need to be passed into the new\nfunction/closure scope.\n\n\nThere is a limitation though: `rss_feeds` is a reference `&`, passed as\nparameter by the `get_feeds()` function caller. The variable is only valid\nin the function scope. Use the following code snippet to provoke this error:\n\n\n```rust\n\npub fn get_feeds(rss_feeds: &HashMap\u003CString, String>) {\n\n    // Store threads in vector\n    let mut threads: Vec\u003Cthread::JoinHandle\u003C()>> = Vec::new();\n\n    // Loop over rss_feeds and spawn threads\n    for (key, value) in rss_feeds {\n        let thread = thread::spawn(move || {\n            println!(\"{}\", key);\n        });\n    }\n}\n\n```\n\n\n![VS Code Terminal, variable scope error with references and thread move\nclosure](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_cargo_build_error_function_threads_variable_scopes.png){:\n.shadow}\n\n\nAlthough the `key` variable was created in the function scope, it references\nthe `rss_feeds` variable, and therefore, it cannot be moved into the thread\nscope. Any values accessed from the function parameter `rss_feeds` hash map\nwill require a local copy with `clone()`.\n\n\n![VS Code Terminal, thread spawn with\nclone](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_rust_thread_spawn_clone.png){:\n.shadow}\n\n\n```rust\n\npub fn get_feeds(rss_feeds: &HashMap\u003CString, String>) {\n\n    // Store threads in vector\n    let mut threads: Vec\u003Cthread::JoinHandle\u003C()>> = Vec::new();\n\n    // Loop over rss_feeds and spawn threads\n    for (name, url) in rss_feeds {\n        let thread_name = name.clone();\n        let thread_url = url.clone();\n        let thread = thread::spawn(move || {\n            // Use thread_name and thread_url as values, see next chapter for instructions.\n```\n\n\n## Parse feed XML into object types\n\nThe next step is to repeat the RSS feed parsing steps in the thread closure.\nAdd the following code comments with instructions for Code Suggestions:\n\n\n1. `// Parse XML body with feed_rs parser, input in bytes` to tell Code\nSuggestions that we want to fetch the RSS feed URL content, and parse it\nwith the `feed_rs` crate functions.\n\n2. `// Check feed_type attribute feed_rs::model::FeedType::RSS2 or Atom and\nprint its name`: Extract the feed type by comparing the `feed_type`\nattribute with the\n[`feed_rs::model::FeedType`](https://docs.rs/feed-rs/latest/feed_rs/model/enum.FeedType.html).\nThis needs more direct instructions for Code Suggestions telling it about\nthe exact Enum values to match against.\n\n\n![Instruct Code Suggestions to match against specific feed\ntypes](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_feed_rs_type_condition.png){:\n.shadow}\n\n\n```rust\n            // Parse XML body with feed_rs parser, input in bytes\n            let body = reqwest::blocking::get(thread_url).unwrap().bytes().unwrap();\n            let feed = feed_rs::parser::parse(body.as_ref()).unwrap();\n\n            // Check feed_type attribute feed_rs::model::FeedType::RSS2 or Atom and print its name\n            if feed.feed_type == feed_rs::model::FeedType::RSS2 {\n                println!(\"{} is an RSS2 feed\", thread_name);\n            } else if feed.feed_type == feed_rs::model::FeedType::Atom {\n                println!(\"{} is an Atom feed\", thread_name);\n            }\n```\n\n\nBuild and run the program again, and verify its output.\n\n\n```\n\ntime cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n\nCNCF is an RSS2 feed\n\nTechCrunch is an RSS2 feed\n\nGitLab Blog is an Atom feed\n\nHacker News is an RSS2 feed\n\n```\n\n\nLet us verify this output by opening the feed URLs in the browser, or\ninspecting the previously downloaded files.\n\n\nHacker News supports RSS version 2.0, with\n`channel(title,link,description,item(title,link,pubDate,comments))`.\nTechCrunch and the CNCF blog follow a similar structure.\n\n```xml\n\n\u003Crss version=\"2.0\">\u003Cchannel>\u003Ctitle>Hacker\nNews\u003C/title>\u003Clink>https://news.ycombinator.com/\u003C/link>\u003Cdescription>Links for\nthe intellectually curious, ranked by\nreaders.\u003C/description>\u003Citem>\u003Ctitle>Writing a debugger from scratch:\nBreakpoints\u003C/title>\u003Clink>https://www.timdbg.com/posts/writing-a-debugger-from-scratch-part-5/\u003C/link>\u003CpubDate>Wed,\n27 Sep 2023 06:31:25\n+0000\u003C/pubDate>\u003Ccomments>https://news.ycombinator.com/item?id=37670938\u003C/comments>\u003Cdescription>\u003C![CDATA[\u003Ca\nhref=\"https://news.ycombinator.com/item?id=37670938\">Comments\u003C/a>]]>\u003C/description>\u003C/item>\u003Citem>\n\n```\n\n\nThe GitLab blog uses the\n[Atom](https://datatracker.ietf.org/doc/html/rfc4287) feed format similar to\nRSS, but still requires different parsing logic.\n\n```xml\n\n\u003C?xml version='1.0' encoding='utf-8' ?>\n\n\u003Cfeed xmlns='http://www.w3.org/2005/Atom'>\n\n\u003C!-- / Get release posts -->\n\n\u003C!-- / Get blog posts -->\n\n\u003Ctitle>GitLab\u003C/title>\n\n\u003Cid>https://about.gitlab.com/blog\u003C/id>\n\n\u003Clink href='https://about.gitlab.com/blog/' />\n\n\u003Cupdated>2023-09-26T00:00:00+00:00\u003C/updated>\n\n\u003Cauthor>\n\n\u003Cname>The GitLab Team\u003C/name>\n\n\u003C/author>\n\n\u003Centry>\n\n\u003Ctitle>Atlassian Server ending: Goodbye disjointed toolchain, hello\nDevSecOps platform\u003C/title>\n\n\u003Clink\nhref='https://about.gitlab.com/blog/atlassian-server-ending-move-to-a-single-devsecops-platform/'\nrel='alternate' />\n\n\u003Cid>https://about.gitlab.com/blog/atlassian-server-ending-move-to-a-single-devsecops-platform/\u003C/id>\n\n\u003Cpublished>2023-09-26T00:00:00+00:00\u003C/published>\n\n\u003Cupdated>2023-09-26T00:00:00+00:00\u003C/updated>\n\n\u003Cauthor>\n\n\u003Cname>Dave Steer, Justin Farris\u003C/name>\n\n\u003C/author>\n\n```\n\n\n### Map generic feed data types\n\nUsing\n[`roxmltree::Document::parse`](https://docs.rs/roxmltree/latest/roxmltree/struct.Document.html)\nwould require us to understand the XML node tree and its specific tag names.\nFortunately,\n[feed_rs::model::Feed](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Feed.html)\nprovides a combined model for RSS and Atom feeds, therefore let us continue\nusing the `feed_rs` crate.\n\n\n1. Atom: Feed->Feed, Entry->Entry\n\n2. RSS: Channel->Feed, Item->Entry\n\n\nIn addition to the mapping above, we need to extract the required\nattributes, and map their data types. It is helpful to open the\n[feed_rs::model\ndocumentation](https://docs.rs/feed-rs/latest/feed_rs/model/index.html) to\nunderstand the structs and their fields and implementations. Otherwise, some\nsuggestions would result in type conversion errors and compilation failures,\nthat are specific to the `feed_rs` implementation.\n\n\nA [`Feed`](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Feed.html)\nstruct provides the `title`, type `Option\u003CText>` (either a value is set, or\nnothing). An\n[`Entry`](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Entry.html)\nstruct provides:\n\n\n1. `title`: `Option\u003CText>`with\n[`Text`](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Text.html) and\nthe `content` field as `String`.\n\n2. `updated`: `Option\u003CDateTime\u003CUtc>>` with\n[`DateTime`](https://docs.rs/chrono/latest/chrono/struct.DateTime.html) with\nthe [`format()`\nmethod](https://docs.rs/chrono/latest/chrono/struct.DateTime.html#method.format).\n\n3. `summary`: `Option\u003CText>`\n[`Text`](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Text.html) and\nthe `content` field as `String`.\n\n4. `links`: `Vec\u003CLink>`, vector with\n[`Link`](https://docs.rs/feed-rs/latest/feed_rs/model/struct.Link.html)\nitems. The `href` attribute provides the raw URL string.\n\n\nUse this knowledge to extract the required data from the feed entries.\nReminder that all `Option` types need to call `unwrap()`, which requires\nmore raw instructions for Code Suggestions.\n\n\n```rust\n                // https://docs.rs/feed-rs/latest/feed_rs/model/struct.Feed.html\n                // https://docs.rs/feed-rs/latest/feed_rs/model/struct.Entry.html\n                // Loop over all entries, and print\n                // title.unwrap().content\n                // published.unwrap().format\n                // summary.unwrap().content\n                // links href as joined string\n                for entry in feed.entries {\n                    println!(\"Title: {}\", entry.title.unwrap().content);\n                    println!(\"Published: {}\", entry.published.unwrap().format(\"%Y-%m-%d %H:%M:%S\"));\n                    println!(\"Summary: {}\", entry.summary.unwrap().content);\n                    println!(\"Links: {:?}\", entry.links.iter().map(|link| link.href.clone()).collect::\u003CVec\u003CString>>().join(\", \"));\n                    println!();\n                }\n```\n\n\n![Code suggestions to print feed entry types, with specific\nrequirements](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_print_feed_entries_fields_with_rust_type_specifics.png){:\n.shadow}\n\n\n### Error handling with Option unwrap()\n\nContinue iterating on the multi-line instructions after building and running\nthe program again. Spoiler: `unwrap()` will call the `panic!` macro and\ncrash the program when it encounters empty values. This can happen if a\nfield like `summary` is not set in the feed data.\n\n\n```shell\n\nGitLab Blog is an Atom feed\n\nTitle: How the Colmena project uses GitLab to support citizen journalists\n\nPublished: 2023-09-27 00:00:00\n\nthread '\u003Cunnamed>' panicked at 'called `Option::unwrap()` on a `None`\nvalue', src/feed_reader.rs:40:59\n\n```\n\n\nA potential solution is to use\n[`std::Option::unwrap_or_else`](https://doc.rust-lang.org/std/option/enum.Option.html#method.unwrap_or_else)\nand set an empty string as default value. The syntax requires a closure that\nreturns an empty `Text` struct instantiation.\n\n\nSolving the problem required many attempts to find the correct\ninitialization, passing just an empty string did not work with the custom\ntypes. I will show you all my endeavors, including the research paths.\n\n\n```rust\n\n// Problem: The `summary` attribute is not always initialized. unwrap() will\npanic! then.\n\n// Requires use mime; and use feed_rs::model::Text;\n\n/*\n\n// 1st attempt: Use unwrap() to extraxt Text from Option\u003CText> type.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap().content);\n\n// 2nd attempt. Learned about unwrap_or_else, passing an empty string.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| \"\").content);\n\n// 3rd attempt. summary is of the Text type, pass a new struct\ninstantiation.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{}).content);\n\n// 4th attempt. Struct instantiation requires 3 field values.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{\"\", \"\",\n\"\"}).content);\n\n// 5th attempt. Struct instantation with public fields requires key: value\nsyntax\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{content_type:\n\"\", src: \"\", content: \"\"}).content);\n\n// 6th attempt. Reviewed expected Text types in\nhttps://docs.rs/feed-rs/latest/feed_rs/model/struct.Text.html and created\nMime and String objects\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{content_type:\nmime::TEXT_PLAIN, src: String::new(), content: String::new()}).content);\n\n// 7th attempt: String and Option\u003CString> cannot be casted automagically.\nCompiler suggested using `Option::Some()`.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{content_type:\nmime::TEXT_PLAIN, src: Option::Some(), content: String::new()}).content);\n\n*/\n\n\n// xth attempt: Solution. Option::Some() requires a new String object.\n\nprintln!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{content_type:\nmime::TEXT_PLAIN, src: Option::Some(String::new()), content:\nString::new()}).content);\n\n```\n\n\nThis approach did not feel satisfying, since the code line is complicated to\nread, and required manual work without help from Code Suggestions. Taking a\nstep back, I reviewed what brought me there - if `Option` is `none`,\n`unwrap()` will throw an error. Maybe there is an easier way to handle this?\nI asked Code Suggestions in a new comment:\n\n\n```\n                // xth attempt: Solution. Option::Some() requires a new String object.\n                println!(\"Summary: {}\", entry.summary.unwrap_or_else(|| Text{content_type: mime::TEXT_PLAIN, src: Option::Some(String::new()), content: String::new()}).content);\n\n                // Alternatively, use Option.is_none()\n```\n\n\n![Code suggestions asked for alternative with\nOptions.is_none](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/code_suggestions_after_complex_unwrap_or_else_ask_for_alternative_option.png){:\n.shadow}\n\n\nIncreased readability, less CPU cycles wasted on `unwrap()`, and a great\nlearning curve from solving a complex problem to using a boring solution.\nWin-win.\n\n\nBefore we forget: Re-add storing the XML data on disk to complete the reader\napp again.\n\n\n```rust\n                // Dump the parsed body to a file, as name-current-iso-date.xml\n                let file_name = format!(\"{}-{}.xml\", thread_name, chrono::Local::now().format(\"%Y-%m-%d-%H-%M-%S\"));\n                let mut file = std::fs::File::create(file_name).unwrap();\n                file.write_all(body.as_ref()).unwrap();\n```\n\n\nBuild and run the program to verify the output.\n\n\n```shell\n\ncargo build\n\n\ntime cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n```\n\n\n![VS Code Terminal, cargo run with formatted feed entries\noutput](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/vs_code_terminal_cargo_run_formatted_output_final.png)\n\n\n## Benchmarks\n\n\n### Sequential vs. Parallel execution benchmark\n\nCompare the execution time benchmarks by creating five samples each.\n\n\n1. Sequential execution. [Example source code\nMR](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai-app-reader/-/merge_requests/1)\n\n2. Parallel exeuction. [Example source code\nMR](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai-app-reader/-/merge_requests/3)\n\n\n```shell\n\n# Sequential\n\ngit checkout sequential-exec\n\n\ntime cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n\n0.21s user 0.08s system 10% cpu 2.898 total\n\n0.21s user 0.08s system 11% cpu 2.585 total\n\n0.21s user 0.09s system 10% cpu 2.946 total\n\n0.19s user 0.08s system 10% cpu 2.714 total\n\n0.20s user 0.10s system 10% cpu 2.808 total\n\n```\n\n\n```shell\n\n# Parallel\n\ngit checkout parallel-exec\n\n\ntime cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\"\n\"CNCF,https://www.cncf.io/feed/\"\n\n\n0.19s user 0.08s system 17% cpu 1.515 total\n\n0.18s user 0.08s system 16% cpu 1.561 total\n\n0.18s user 0.07s system 17% cpu 1.414 total\n\n0.19s user 0.08s system 18% cpu 1.447 total\n\n0.17s user 0.08s system 16% cpu 1.453 total\n\n```\n\n\nThe CPU usage increased for parallel execution of four RSS feed threads, but\nit nearly halved the total time compared to sequential execution. With that\nin mind, we can continue learning Rust and optimize the code and\nfunctionality.\n\n\nNote that we are running the debug build through Cargo, and not the\noptimized released builds yet. There are caveats with parallel execution\nthough: Some HTTP endpoints put rate limits in place, where parallelism\ncould hit these thresholds easier.\n\n\nThe system executing multiple threads in parallel might get overloaded too –\nthreads require context switching in the Kernel, assigning resources to each\nthread. While one thread gets computing resources, other threads are put to\nsleep. If there are too many threads spawned, this might slow down the\nsystem, rather than speeding up the operations. Solutions include design\npatterns such as [work\nqueues](https://docs.rs/work-queue/latest/work_queue/), where the caller\nadds a task into a queue, and a defined number of worker threads pick up the\ntasks for asynchronous execution.\n\n\nRust also provides data synchronisation between threads, so-called\n[channels](https://doc.rust-lang.org/rust-by-example/std_misc/channels.html).\nTo ensure concurrent data access,\n[mutexes](https://doc.rust-lang.org/std/sync/struct.Mutex.html) are\navailable to provide safe locks.\n\n\n### CI/CD with Rust caching\n\nAdd the following CI/CD configuration into the `.gitlab-ci.yml` file. The\n`run-latest` job calls `cargo run` with RSS feed URL examples, and measures\nthe execution time continuously.\n\n\n```\n\nstages:\n  - build\n  - test\n  - run\n\ndefault:\n  image: rust:latest\n  cache:\n    key: ${CI_COMMIT_REF_SLUG}\n    paths:\n      - .cargo/bin\n      - .cargo/registry/index\n      - .cargo/registry/cache\n      - target/debug/deps\n      - target/debug/build\n    policy: pull-push\n\n# Cargo data needs to be in the project directory for being cached.\n\nvariables:\n  CARGO_HOME: ${CI_PROJECT_DIR}/.cargo\n\nbuild-latest:\n  stage: build\n  script:\n    - cargo build --verbose\n\ntest-latest:\n  stage: build\n  script:\n    - cargo test --verbose\n\nrun-latest:\n  stage: run\n  script:\n    - time cargo run -- \"GitLab Blog,https://about.gitlab.com/atom.xml\" \"CNCF,https://www.cncf.io/feed/\"\n```\n\n\n![GitLab CI/CD pipelines for Rust, cargo run\noutput](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-advanced-programming/gitlab_cicd_pipeline_rust_cargo_run_output.png){:\n.shadow}\n\n\n## What is next\n\nThis blog post was challenging to create, with both learning advanced Rust\nprogramming techniques myself, and finding a good learning curve with Code\nSuggestions. The latter greatly helps with quickly generating code, not just\nboilerplate snippets – it understands the local context, and better\nunderstands the purpose and scope of the algorithm, the more code you write.\nAfter reading this blog post, you know of a few challenges and turnarounds.\nThe example solution code for the reader app is available in [the\nlearn-rust-ai-app-reader\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai-app-reader).\n\n\nParsing RSS feeds is challenging since it involves data structures, with\nexternal HTTP requests and parallel optimizations. As an experienced Rust\nuser, you might have wondered: `Why not use the std::rss crate?` -- It is\noptimized for advanced asynchronous execution, and does not allow to show\nand explain the different Rust functionalities, explained in this blog post.\nAs an async exercise, try to rewrite the code using the [`rss`\ncrate](https://docs.rs/rss/latest/rss/).\n\n\n### Async learning exercises\n\nThe lessons learned in this blog post also lay the foundation for future\nexploration with persistent storage and presenting the data. Here are a few\nideas where you can continue learning Rust and optimize the reader app:\n\n\n1. Data storage: Use a database like sqlite, and RSS feed update tracking.\n\n2. Notifications: Spawn child processes to trigger notifications into\nTelegram, etc.\n\n3. Functionality: Extend the reader types to REST APIs\n\n4. Configuration: Add support for configuration files for RSS feeds, APIs,\netc.\n\n5. Efficiency: Add support for filters, and subscribed tags.\n\n6. Deployments: Use a webserver, collect Prometheus metrics, and deploy to\nKubernetes.\n\n\nIn a future blog post, we will discuss some of these ideas, and how to\nimplement them. Dive into existing RSS feed implementations, and learn how\nyou can refactor the existing code into leveraging more Rust libraries\n(`crates`).\n\n\n### Share your feedback\n\nWhen you use [GitLab Duo](/gitlab-duo/) Code Suggestions, please [share your\nthoughts in the feedback\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/405152).\n",[851,1121,9,956,720],{"slug":3209,"featured":6,"template":700},"learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions","content:en-us:blog:learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions.yml","Learn Advanced Rust Programming With A Little Help From Ai Code Suggestions","en-us/blog/learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions.yml","en-us/blog/learn-advanced-rust-programming-with-a-little-help-from-ai-code-suggestions",{"_path":3215,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3216,"content":3222,"config":3228,"_id":3230,"_type":13,"title":3231,"_source":15,"_file":3232,"_stem":3233,"_extension":18},"/en-us/blog/learn-gitlab-devops-version-control",{"title":3217,"description":3218,"ogTitle":3217,"ogDescription":3218,"noIndex":6,"ogImage":3219,"ogUrl":3220,"ogSiteName":685,"ogType":686,"canonicalUrls":3220,"schema":3221},"GitLab tutorials for secure pipelines, Kubernetes, and more at Learn@GitLab","Learn@GitLab offers videos and self-driven demos so you can get the most out of GitLab at your own pace.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667236/Blog/Hero%20Images/Learn-at-GL.jpg","https://about.gitlab.com/blog/learn-gitlab-devops-version-control","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"GitLab tutorials for secure pipelines, Kubernetes, and more at Learn@GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Chrissie Buchanan\"}],\n        \"datePublished\": \"2021-01-12\",\n      }",{"title":3217,"description":3218,"authors":3223,"heroImage":3219,"date":3225,"body":3226,"category":785,"tags":3227},[3224],"Chrissie Buchanan","2021-01-12","\nAt GitLab, we often say that it's not what you know, it's knowing where to look. But sometimes, finding answers isn’t so easy.\n\nAn autonomous, [self-service](/company/culture/all-remote/self-service/#proactive-approach-to-answering-questions), self-learning, and self-searching mindset is when you operate with the idea that your question has already been answered – somewhere. But we realized that for people interested in GitLab, or even those using GitLab, learning **how** to use it wasn’t always easy to find.\n\nWhile we stress the importance of having a [single source of truth](https://handbook.gitlab.com/handbook/values/#single-source-of-truth), we realized that when it came to learning about GitLab, there were almost too many places to look. We have [GitLab University](https://docs.gitlab.com/ee/index.html), our official [GitLab](https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg) and [GitLab Unfiltered](https://www.youtube.com/channel/UCMtZ0sc1HHNtGGWZFDRTh5A) YouTube pages where we regularly upload educational content, and of course, the [docs](https://docs.gitlab.com/). We needed to find a way to consolidate self-education and make it more intuitive.\n\n## What is Learn@GitLab?\n\n[Learn@GitLab](/learn/) is a learning portal where anyone can go to find self-driven demos and videos about using GitLab. Rather than just making Learn@GitLab _one more resource_, we’re iterating on this idea and consolidating our educational content so that it’s self-driven and easy to find.\n\nThe goal for Learn@GitLab is to present high quality, and accessible technical content that is easy to find on our website to help prospects and users educate themselves about GitLab. This content will include educational technical videos, as well as simulation/click-through demos, and tutorials. The content is organized by common topics such as [DevOps Platform](/solutions/devops-platform/), [version control](/topics/version-control/) and collaboration, and continuous integration, to name a few.\n\nWe’ve picked three of our favorite videos/tutorials for you to get a quick introduction to Learn@GitLab.\n\n## The benefits of a single DevOps platform\n\nWhen we talk about the benefits of GitLab, we often talk about how it saves time and how the single application reduces toolchain complexity. But what does that mean in the context of an ordinary toolchain using tools like GitHub, Jenkins, Jira, etc.?\n\nIn this super short video, we break down a typical toolchain according to three criteria: Integrations needed, clicks, and screen switches. How many times do you need to context switch for a simple task? We break it down for you.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/MNxkyLrA5Aw\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## Adding security to your GitLab CI/CD pipeline\n\nGitLab helps teams go from DevOps to DevSecOps. One of the ways we help is by allowing you to check your application for security vulnerabilities in your CI/CD pipelines that may lead to unauthorized access, data leaks, denial of services, or worse. GitLab reports these vulnerabilities in the merge request so you can fix them before they ever reach end users.\n\nThis quick video guides you through setting up and configuring GitLab security features, and setting up approval rules for merge requests.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/Fd5DhebtScg\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## GitLab developer flow with Kubernetes\n\nIf you’re a developer, or even just managing a team of developers, you might want to see what a typical workflow would be like using GitLab. If you’re using [Kubernetes](/solutions/kubernetes/), seeing how GitLab works within a deployment environment is especially important.\n\nIn this technical demo, we use Amazon EKS as the deployment environment. We go over creating GitLab issues, merge requests, how to use Auto DevOps pipeline templates, review apps, advanced deployment techniques, and staging and production rollout – all in **just 15 minutes.**\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/TMQziI2VDbQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nWhile we’ll continue to have educational content in other places on our site (and will continue to update them), Learn@GitLab will act as a front door for self education that is no more than two clicks from our homepage. With this new learning portal, we hope to teach people what problems GitLab can solve, but more importantly, show step-by-step _how_ GitLab solves them.\n\nFeel free to explore the different learning paths and comment below if you have any suggestions. Everyone can contribute.\n\n\u003Ci class=\"fab fa-gitlab\" style=\"color:rgb(107,79,187); font-size:.85em\" aria-hidden=\"true\">\u003C/i>  \n[Go to Learn@GitLab](/learn/)!\n  \u003Ci class=\"fab fa-gitlab\" style=\"color:rgb(107,79,187); font-size:.85em\" aria-hidden=\"true\">\u003C/i>\n\nCover image by [Benjamin Davies](https://unsplash.com/@bendavisual?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText) on [Unsplash](https://unsplash.com/s/photos/learn?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\n",[1263,232,9],{"slug":3229,"featured":6,"template":700},"learn-gitlab-devops-version-control","content:en-us:blog:learn-gitlab-devops-version-control.yml","Learn Gitlab Devops Version Control","en-us/blog/learn-gitlab-devops-version-control.yml","en-us/blog/learn-gitlab-devops-version-control",{"_path":3235,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3236,"content":3241,"config":3248,"_id":3250,"_type":13,"title":3251,"_source":15,"_file":3252,"_stem":3253,"_extension":18},"/en-us/blog/learn-python-with-pj-part-3",{"title":3237,"description":3238,"ogTitle":3237,"ogDescription":3238,"noIndex":6,"ogImage":1112,"ogUrl":3239,"ogSiteName":685,"ogType":686,"canonicalUrls":3239,"schema":3240},"Learn Python with Pj! Part 3 - Functions and strings","Pj shares his experiences learning how to program functions and strings.","https://about.gitlab.com/blog/learn-python-with-pj-part-3","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learn Python with Pj! Part 3 - Functions and strings\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"PJ Metz\"}],\n        \"datePublished\": \"2022-04-04\",\n      }",{"title":3237,"description":3238,"authors":3242,"heroImage":1112,"date":3244,"body":3245,"category":785,"tags":3246},[3243],"PJ Metz","2022-04-04","\n\n_This is the third installment in the Learn Python with Pj! series. Make sure to read [Part 1](/blog/learn-python-with-pj-part-1/) and [Part 2](/blog/learn-python-with-pj-part-2/)._\n\n\nIn learning Python, I’m happy to have found a language with a straightforward syntax that just seems to make sense. I don’t have to define a type; Python just knows. I don’t have to worry about `let` or `const` or `var` for different use cases; I just make the variable. I’m very glad I learned C# and JavaScript first, as those feel important to understanding exactly what is happening when I write code. In turn, I think it’s made Python easier for me, which is usually true when learning another programming language: Your second and third are easier to learn since your brain understands “programming logic” better now than when you made your first “Hello World”. This week we’re going to talk about what I’ve learned in functions and strings. \n\n## Functions\n\nFunctions are the backbone of any app you write. It’s an important step in learning any language to learn how to put a series of actions inside a single function that can be called later in the code. Python does this simply compared to other languages I’ve learned. \n\n```python\ndef my_first_function(arg1, arg2):\n  print(f”Your input was {arg1} and {arg2}.”)\n\n#prints “Your input was hilarious and unnecessary”\nmy_first_function(“hilarious”, “unnecessary”)\n```\nUsing the keyword `def` lets Python know that you’re about to write a function. Inside the parentheses, you put any parameters that must be included when calling the function. Some people use argument and parameter interchangeably, but technically, when defining a function, it’s a parameter, and when calling a function, it’s an argument. Either way, when defining the function, include some variables that you’ll expect when the function is called later. Finally, put a colon and then move to the next line. All the code for the function to run is indented. Inside the function, you can run loops, logic, or even other functions. Let’s check out a slightly more complex use. \n\n```python\ndef halloween_horror_nights(days, link,):\n    named = input(\"What is your name?\")\n    name = named.capitalize()\n    if isinstance(days, int):\n        if days == 0:\n            print(f\" Hello, {name}. We're ready to see you at HHN. {link}\")\n        elif days \u003C= 30:\n            print(f\"{name}, You have {days} days until the terror is home. {link}\")\n        elif days \u003C= 60:\n            print(f\"{name}, The horror comes home in {days} days. Join us in the dark. {link}\")\n        elif days \u003C 365 and days > 60:\n            print(f\"{name}, Patience is a virtue. You're {days} days away from the top rated Halloween event in the world.{link}\")\n        else:\n            print(f\"{name}, it can't be more than a year away. It's closer than you think... {link}\")\n    else:\n        print(\"Days must be an int\")\n    \n    \n#This will print “{Name input by user}, The horror comes home in 56 days. Join us in the dark. https://orlando.halloweenhorrornights.com/site\"\nhalloween_horror_nights(56, \"https://orlando.halloweenhorrornights.com/site\")\n```\n\nFor context, Halloween Horror Nights in Orlando is my favorite event of the year. This function takes in a number of days and a link (meant to be days until the event and a link to the HHN web page) and outputs a string that says how many days are left until the event. The string also includes a link to the web page and asks for user input to personalize each string. The function `isinstance()` checks if `days` is an int to make sure the sentence makes sense and returns `True` if the first argument is the type of the second argument. \n\nI really found functions in Python to be a lot easier than in other languages, though I still miss the curly brackets of C# and JavaScript. Additionally, the simplicity of `def` followed by the function name and any required parameters is really straightforward and makes reading the code easier. And since code is read more than it’s written, that makes Python pretty awesome in my book. \n\nI also used the f-string format for these print statements, and it’s still one of my favorite ways to concatenate. It feels easier than a lot of the other ways of inserting variables into a string in Python, and a little easier than the way it’s done in JavaScript, at least to me. I use a different method of including variables in a string called `.format()`. \n\nMaking your own functions is important, but there are a bunch of built-in functions in Python. There are also methods, which are similar to functions but are associated with the objects in a class they’re assigned to. Let’s talk about some strings and some methods that come with them.\n\n## Strings\n\nI thought it was strange that I had a whole section on strings in my Codecademy Python curriculum, but I soon realized that it was giving me a lot of very useful methods to use on strings that seemed very versatile. The most interesting thing to me is that strings are an object and act like a list of characters. I’m not entirely sure how strings are treated in other languages, but this really struck me as a cool idea. You can even call specific characters using the same syntax you would for a list. \n\n```python\nspooky = “Halloween Horror Nights is my favorite thing about Autumn.”\n\n#the following prints “l” since it’s the 3rd char in the string `spooky`. \nprint(spooky[2]) \n```\n\nOr you can use a for loop on a string.\n\n```python\n#This prints each letter on a new line and capitalizes it. The message now reads vertical in the output.\nfor letter in spooky:\n  print(letter.upper())\n```\n\n### String methods\t\n\nA few built-in methods exist for strings in Python, like `capitalize()` and `upper()`, two I used in the above examples. In addition to those, there are many more that can do things like remove the whitespace or noise from the beginning and end of a string, tell you the index of the first appearance of something, or join a list of strings into a single string. There are lots of great included methods. Here’s an exercise I took from Codecademy and changed the content to fit this article's theme. \n\n```python\n#given a string that contains a ton of information separated by semicolons and commas. Each part is a haunted house name, Universal Studios location, and the year the house appeared at the event.\nhhn_houses_location_year = \"Chucky;Japan;2016, Run;Orlando;2001, The Orfanage: Ashes to Ashes;Orlando;2010, The Real: Haunted Village;Japan;2021, The Undertaker: No Mercy;Hollywood;2000, Welcome to Silent Hill;Hollywood;2012, American Werewolf in London;Orlando;2013\"   \n\n#this splits the string up into a list where each element of the list is the section separated by a comma.\nhhn_houses_list = hhn_houses_location_year.split(\",\")\n\n#empty list for the next step\nhhn_houses_stripped = []\n\n#this strips any whitespace from the element in the list and adds it to the empty list from before\nfor house in hhn_houses_list:\n    hhn_houses_stripped.append(house.strip())\n\n#empty list for next step\nhhn_house_details = []\n\n#the next few lines split the details into their own list. \n#first, each house, with the details, is split along the semicolons to make a list of lists, with each house being its own element in the larger list\n#next, empty lists are made for each detail\n#finally, using index numbers, each detail is placed in it’s own list so all the houses, locations, and years are separated. \nfor info in hhn_houses_stripped:\n    hhn_house_details.append(info.split(\";\"))\n\nhouse = []\nlocation = []\nyear = []\n\nfor stuff in hhn_house_details:\n    house.append(stuff[0])\n    location.append(stuff[1])\n    year.append(stuff[2])\n\n#loops through and using .format() prints a sentence that tells about each house. \nfor num in range(0, len(house)):\n  print(\"{} was located in {} for the {} event\".format(house[num], location[num], year[num]))\n```\n\nAs you can see, I am obsessed with Halloween horror nights… er, wait, not the point of the article. As you can see, Python’s built-in methods for strings can be pretty useful, especially if you end up with a bunch of data sitting around in unformatted strings. Next time, we’re going to talk about Dictionaries and how they are used in Python! \n",[1121,3247,9],"growth",{"slug":3249,"featured":6,"template":700},"learn-python-with-pj-part-3","content:en-us:blog:learn-python-with-pj-part-3.yml","Learn Python With Pj Part 3","en-us/blog/learn-python-with-pj-part-3.yml","en-us/blog/learn-python-with-pj-part-3",{"_path":3255,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3256,"content":3261,"config":3266,"_id":3268,"_type":13,"title":3269,"_source":15,"_file":3270,"_stem":3271,"_extension":18},"/en-us/blog/learn-python-with-pj-part-4-dictionaries-and-files",{"title":3257,"description":3258,"ogTitle":3257,"ogDescription":3258,"noIndex":6,"ogImage":1112,"ogUrl":3259,"ogSiteName":685,"ogType":686,"canonicalUrls":3259,"schema":3260},"Learn Python with Pj! Part 4 - Dictionaries and Files","Our education evangelist Pj Metz continues his journey to learn how to code in Python.","https://about.gitlab.com/blog/learn-python-with-pj-part-4-dictionaries-and-files","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learn Python with Pj! Part 4 - Dictionaries and Files\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"PJ Metz\"}],\n        \"datePublished\": \"2022-05-05\",\n      }",{"title":3257,"description":3258,"authors":3262,"heroImage":1112,"date":3263,"body":3264,"category":1121,"tags":3265},[3243],"2022-05-05","This is the fourth installment in the Learn Python with Pj! series. Make\nsure to read:\n\n- [Part 1 - Getting started](/blog/learn-python-with-pj-part-1/)\n\n- [Part 2 - Lists and loops](/blog/learn-python-with-pj-part-2/)\n\n- [Part 3 - Functions and\nstrings](/blog/learn-python-with-pj-part-3/)\n\n- [Part 5 - Build a hashtag tracker with the Twitter\nAPI](/blog/learn-python-with-pj-part-5-building-something-with-the-twitter-api/)\n\n\nI’ve learned a lot with Python so far, but when I learned dictionaries\n(sometimes shortened to dicts), I was really excited about what could be\ndone. A dictionary in Python is a series of keys and values stored inside a\nsingle object. This is kind of like a super array; one that allows you to\nconnect keys and values together in a single easily accessible source.\nCreating dictionaries from arrays can actually be very simple, too.\n\n\nIn this blog, I'll dig into how to create dictionaries and how to read and\nwrite files in the code.\n\n\n## Dictionaries\n\n\nDictionaries in Python are indicated by using curly braces, or as I like to\ncall them, mustaches. `{ }` indicates that the list you’re looking at isn’t\na list at all, but a dictionary. \n\n\n```python\n\nshows_and _characters = {\n    \"Bojack Horseman\": \"Todd\",\n    \"My Hero Academia\": \"Midoriya\"\n    \"Ozark\": \"Ruth\"\n    \"Arrested Development\": \"Tobias\",\n    \"Derry Girls\": \"Sister Michael\",\n    \"Tuca & Bertie\": \"Bertie\"\n    }\n```\n\n\nThis is a dictionary of my favorite TV shows and my favorite characters in\nthat show. In this example, the key is on the left and the value is on the\nright. To access dictionaries, you use a similar call like you would for a\nlist, except instead of an element number, you would put the key.\n`print(shows_and_characters[“Ozark”])` would print `Ruth` to the console.\nAdditionally, both the key and value in this example are strings, but that’s\nnot a requirement. Keys can be any immutable type, like strings, ints,\nfloats, and tuples. Values don’t have this same restriction, therefore\nvalues can be a nested dictionary or a list, in addition to the types\nmentioned for keys. For instance, the following dictionary is a valid\ndictionary.\n\n\n```python\n\nshows_with_lists = {\n    \"Bojack Horseman\": [\"Todd\", \"Princess Carolyn\", \"Judah\", \"Diane\"],\n    \"My Hero Academia\": [\"Midoriya\", \"Shoto\", \"All Might\", \"Bakugo\", \"Kirishima\"],\n    \"Ozark\": [\"Ruth\", \"Jonah\", \"Wyatt\"],\n    \"Arrested Development\": [\"Tobias\", \"Gob\", \"Anne\", \"Maeby\"],\n    \"Derry Girls\": [\"Sister Michael\", \"Orla\", \"Erin\", \"Claire\", \"James\"],\n    \"Tuca & Bertie\": [\"Bertie\", \"Speckle\", \"Tuca\", \"Dakota\"]\n    }\n```\n\nIn this example, each value is a list. So if we tried to print the value for\nthe key `”Derry Girls”`, we would see `[“Sister Michael”, “Orla”, “Erin”,\n“Claire”, “James”]` printed to the console. However, if we wanted the last\nelement in the value list, we’d write `shows_with_lists[“Derry Girls”]\n[-1]`. This would print the last element in the list, which in this case is\n`James`. \n\n\nDictionaries can be written manually, or, if you have two lists, you can\ncombine the `dict()` and `zip()` methods to make the lists into a\ndictionary. \n\n\n```python\n\nlist_of_shows = [\"Bojack Horseman\",\n                 \"My Hero Academia\",\n                 \"Ozark\",\n                 \"Arrested Development\",\n                 \"Derry Girls\",\n                 \"Tuca & Bertie\"]\nlist_of_characters = [[\"Todd\", \"Princess Carolyn\", \"Judah\", \"Diane\"],\n                      [\"Midoriya\", \"Shoto\", \"All Might\", \"Bakugo\", \"Kirishima\"],\n                      [\"Ruth\", \"Jonah\", \"Wyatt\"],\n                      [\"Tobias\", \"Gob\", \"Anne\", \"Maeby\"],\n                      [\"Sister Michael\", \"Orla\", \"Erin\", \"Claire\", \"James\"],\n                      [\"Bertie\", \"Speckle\", \"Tuca\", \"Dakota\"]]\n\ncombined_shows_characters = dict(zip(list_of_shows, list_of_characters))\n\n\nprint(combined_shows_characters)\n\n```\n\n\nThis is one way to create a dictionary. Another is called Dictionary\nComprehension. This one is a little more work, but can be used in a variety\nof different ways, including using a bit of logic on a single list to\ngenerate a dictionary using that original list. Here’s how with two\nexamples: one based on the above lists, and one with a single list and some\nlogic. \n\n\n```python\n\nimport math\n\n\n#This is doing the same work as the above example, but using Dict\nComprehension instead. \n\ncomprehension_shows_characters = { shows:characters for shows, characters in\nzip(list_of_shows, list_of_characters)  }\n\n\nhip_to_be_square = [4, 9, 16, 25, 36, 49]\n\n\nno_longer_hip_to_be_square = { key:math.sqrt(key) for key in\nhip_to_be_square }\n\n\nprint(no_longer_hip_to_be_square)\n\n```\n\n\nIn the `no_longer_hip_to_be_square` dictionary, the key is found in the\n`hip_to_be_square` list. The value for each key is its own square root,\nbrought in with the import math function. There are plenty more useful\nmethods for dealing with dictionaries\n[here](https://realpython.com/python-dicts/). \n\n\n## Reading and writing files\n\n\nThis one is a pretty cool part of Python: reading and writing other files\nright in the code. With Python, you’re able to take the contents of certain\ntypes of files and use it in your code, or even create a new file based on\nsome input. This is useful for data handling and can be used with a  variety\nof file types. The two I’ll be covering here are .csv and .txt.\n\n\n### Reading from a file\n\n\nImagine a .txt file named `best-ever.txt` containing the line `My favorite\ntv show is Derry Girls`. We can use Python to take that line and turn it\ninto a variable. Running the following code would print the contents of the\n.txt file to the terminal. \n\n\n```python\n\nwith open(\"best-ever.txt\") as text_file:\n  text_data = text_file.read()\n\n#This will print the contents of the .txt file. \n\nprint(text_data)\n\n```\n\n\nBy using `with open(NAME OF FILE) as VARIABLE_NAME:`, we can examine the\ncontents of files as a single string. If the document has multiple lines,\nyou can even separate those by iterating over them by using a for loop and\nthe `.readlines()` method. Using an imaginary .txt document called\n`buncha-lines` we could use the following to print out each line\nindividually.\n\n\n```python\n\nwith open(\"buncha-lines.txt\") as lines_doc:\n  for line in lines_doc.readlines():\n    print(line)\n``` \n\n### Writing a new file\n\n\nCreating a new file is also easy with Python. The `open()` function can take\nan additional argument in order to create a new file. In fact, there’s a\ndefault argument that’s been being passed each time without us knowing! `r`\nis the default argument for `open()` and puts it in read mode. To turn on\nwrite mode, pass in a `w` as the second argument. The following code will\nwrite a brand-new file called `best_tv_character.txt` with the contents\n`Peggy Olson from Mad Men`. \n\n\n```python\n\nwith open(\"best_tv_character.txt\", \"w\") as best_character:\n  best_character.write(\"Peggy Olson from Mad Men\")\n```\n\n### Working with .csv files\n\n\nYou can read a .csv file with Python by using `import csv` at the beginning\nof the file, and then using some of its built-in methods in the code.\nHowever, even though .csv files are plain text, treating a .csv file the\nsame as you treat .txt files can lead to difficult to read outputs; after\nall, the point of a spreadsheet is to table information. Without that table,\nthe output can be chaotic. A way around this is to use the `dictreader()`\nmethod. This method allows you to map the information in each row to a\ndictionary with field names you can create. The default field names are\ncollected from the first row of the .csv if no field names are given.\nImagine a .csv file with columns labeled, “Network”, “Show name”, “Seasons”.\nMaybe we just want to print the number of seasons from this .csv. \n\n\n```python\n\nimport csv \n\n\nwith open(\"shows.csv\") as shows_csv:\n  shows_dict = csv.DictReader(shows_csv)\n  for row in shows_dict:\n    print(row[\"Seasons\"])\n```\n\n\nThis would print to the console, on a new line, the number of seasons for\neach row that exists in the .csv. \n\n\nJust like with .txt files, you can also create .csv files with Python. It’s\na bit more complicated since you need to define the headers, or column\nnames, but it is still a quick process. This can be used to take lists and\nturn them into .csv files. Let’s check out the following example:\n\n\n```python\n\nimport csv\n\n\nworking_list = [{\"Network\": \"Netflix\", \"Show Name\":\"Bojack Horseman\",\n\"Seasons\":6}, {\"Network\":\"Channel 4\",\"Show Name\":\"Derry Girls\", \"Seasons\":\n3}, {\"Network\":\"HBO Max\", \"Show Name\":\"Our Flag Means Death\", \"Seasons\": 1}]\n\n\n\nwith open(\"shows.csv\", \"w\") as shows_csv:\n    fields = [\"Network\", \"Show Name\", \"Seasons\"]\n    shows_w = csv.DictWriter(shows_csv, fieldnames = fields)\n\n    shows_w.writeheader()\n    for item in working_list:\n        shows_w.writerow(item)\n```\n\n\nThis previous code block creates a brand-new csv file by using the `”w”`\nparameter in `open()`. We manually name the fields in the order they appear\nin a separate list, then pass that list into the `DictWriter` parameter\n`fieldnames`. Finally, we use the `writeheader()` and a for loop with the\n`writerow()` methods to create a header row and to iterate over the\n`working_list` and turn each entry into a row in the .csv. \n\n\nThese are only a few ways to work with .csv and .txt files; Python is very\nversatile and more information [can be found\nhere](https://realpython.com/working-with-files-in-python/).\n",[1121,9,3247],{"slug":3267,"featured":6,"template":700},"learn-python-with-pj-part-4-dictionaries-and-files","content:en-us:blog:learn-python-with-pj-part-4-dictionaries-and-files.yml","Learn Python With Pj Part 4 Dictionaries And Files","en-us/blog/learn-python-with-pj-part-4-dictionaries-and-files.yml","en-us/blog/learn-python-with-pj-part-4-dictionaries-and-files",{"_path":3273,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3274,"content":3279,"config":3284,"_id":3286,"_type":13,"title":3287,"_source":15,"_file":3288,"_stem":3289,"_extension":18},"/en-us/blog/learn-python-with-pj-part-5-building-something-with-the-twitter-api",{"title":3275,"description":3276,"ogTitle":3275,"ogDescription":3276,"noIndex":6,"ogImage":1112,"ogUrl":3277,"ogSiteName":685,"ogType":686,"canonicalUrls":3277,"schema":3278},"Learn Python with Pj! Part 5 - Build a hashtag tracker with the Twitter API","Our Education Evangelist Pj Metz wraps up his five-part series with this penultimate tutorial.","https://about.gitlab.com/blog/learn-python-with-pj-part-5-building-something-with-the-twitter-api","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learn Python with Pj! Part 5 - Build a hashtag tracker with the Twitter API\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"PJ Metz\"}],\n        \"datePublished\": \"2022-06-01\",\n      }",{"title":3275,"description":3276,"authors":3280,"heroImage":1112,"date":3281,"body":3282,"category":1121,"tags":3283},[3243],"2022-06-01","\nThis is the fifth and final installment in the Learn Python with Pj! series. Make sure to read:\n- [Part 1 - Getting started](/blog/learn-python-with-pj-part-1/)\n- [Part 2 - Lists and loops](/blog/learn-python-with-pj-part-2/)\n- [Part 3 - Functions and strings](/blog/learn-python-with-pj-part-3/)\n- [Part 4 - Dictionaries and Files](/blog/learn-python-with-pj-part-4-dictionaries-and-files/)\n\n## Putting it all together\nI’ve completed my Python course on [Codecademy](https://codecademy.com/), and am excited to put the skills I learned into building something practical. I’ve worked with the Twitter API before; I wrote a few bots in Node.js to make them tweet and respond to tweets they’re tagged in. I thought it’d be fun to work with the API again, but this time do it in Python. I didn’t just want to make another bot, so I had to figure out something else. In this case, I made a bot that can track hashtags being used in real time on Twitter.\n\nHere’s [my repo](https://gitlab.com/MetzinAround/python-hashtagger) containing a few different files, but `live_tweets.py` is what we’ll focus on for this blog. Let’s talk about how I built it and what it does. \n\n```python\nimport tweepy\nimport config\n\nauth = tweepy.OAuth1UserHandler(config.consumer_key, config.consumer_secret, config.access_token, config.access_token_secret\n)\n\napi = tweepy.API(auth) \n\n#prints the text of the tweet using hashtag designated in stream.filter(track=[])\nclass LogTweets(tweepy.Stream):\n        def on_status(self, status):\n                date = status.created_at\n                username = status.user.screen_name\n                \n                try:\n                        tweet = status.extended_tweet[\"full_text\"]\n                except AttributeError:\n                        tweet = status.text\n\n                print(\"**Tweet info**\")\n                print(f\"Date: {date}\")\n                print(f\"Username: {username}\")\n                print(f\"Tweet: {tweet}\")\n                print(\"*********\")\n                print(\"********* \\n\")\n              \n\nif __name__ == \"__main__\":         \n        #creates instance of LogTweets with authentication\n        stream = LogTweets(config.consumer_key, config.consumer_secret, config.access_token, config.access_token_secret)\n\n\n        #hashtags as str in list will be watched live on twitter. \n        hashtags = []\n        print(\"Looking for Hashtags...\")\n        stream.filter(track=hashtags)\n\n\n```\n\nHere’s how this all works. First, we import two modules: [Tweepy](https://www.tweepy.org/) and config. Tweepy is a wrapper that makes using the Twitter API very easy. Config allows us to use config files and keep our secrets safe. This is important since using the Twitter API involves four keys that are specific to your Twitter developer account. Getting these keys is covered in this Twitter [documentation](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api). We’ll talk about what’s in the config file and how it works later. \n\nThe next line defines the variable `auth` using tweepy’s built in authorization handler. Normally, you’d put in the keys directly here, but since we’re trying to keep secrets safe, we handle those through the config file. In order to call those variables hosted in the config file, we type `config.variable_name`. Finally, in order to access the tweepy api, we create the variable `api` with the auth variable from the line above passed into `tweepy.API()`. Now, the variable `api` will give us access to all the features in Tweepy’s Twitter API library. \n\n> You’re invited! Join us on June 23rd for the [GitLab 15 launch event](https://page.gitlab.com/fifteen) with DevOps guru Gene Kim and several GitLab leaders. They’ll show you what they see for the future of DevOps and The One DevOps Platform.\n\nFor our purposes, we want to find a hashtag being used, then collect the tweet that used it and print some information about the tweet to the console. To make this happen, we’ve created a class called `LogTweets` that takes an input `tweepy.Stream`. Stream is a Twitter API term that refers to all of the tweets being posted on Twitter at any given moment. Think of it as opening a window looking out onto every single tweet as it’s posted. We have to make this open connection in order to be able to find tweets that are using our hashtag. Inside `LogTweets`, we define a function called `on_status` with the parameters `self` and `status`. `On_status` will be called when a status is detected in the stream. `Self` is required as the first parameter in any class function, and `status` in this function will be referring to the status posted by a Twitter user, often called a tweet.\n\nIn our case, we’re going with status because `tweet` will represent the text of the status itself. We define `date` and `username` using Tweepy documentation: `created_at` is the date and `user.screen_name` is the username of the person who posted the status.\n\nNext is a `try/except` block. Try/except is a concept that works similarly to an if statement, but it allows for error handling a little bit better. It essentially says, “Try this, but if there’s a problem, do this instead.” In this case, we try to define the variable `tweet` as `.extended_tweet[“full_text”]`. This checks if the status we’re working with has the `extended_tweet` attribute. Twitter used to be limited to 140 characters, and when they increased the limit to 280, the `extended_tweet` became necessary.\n\nNow, if you want to capture the full tweet, you need the `extended_tweet` attribute. Inside of that attribute is the key `full_text`. Longer tweets will need that full_text or it will cut off at the 140 character limit. This `try` command checks if that key exists; if it does, `tweet` is equal to that full text.\n\nHowever, if an `AttributeError` happens, we just grab the regular text and set it equal to the variable `tweet`. Next, we print some info to the terminal. Whenever this function is called, the six lines will print to the console with the variables created above replaced by whatever status info was passed in. This makes it easier to keep track of what we’re looking at in the terminal. \n\nNext, we have an important if statement: `if __name__ == \"__main__\":`. This is used to indicate what happens when the file is run. Basically, files in Python receive a property called `__name__` from the compiler. The file that is called to be run directly is called `__main__`. Other files not run are given names equal to the file name. Therefore, anything under this if statement will only run if the file is being called directly by the compiler. \n\nNext, we create an instance of `LogTweets` called `stream`. We pass in the authentication information from the config file just like we did for the `auth` variable in the beginning of the code. This “opens up” the stream and we are now looking at all the tweets being sent in real time. In order to narrow our search, we need something to look for. The variable `hashtags` is an empty list that must be populated with strings of the hashtags we’re looking to track. This list will be put into the keyword `track` in a few lines. \n\n`Track` is an important keyword for the stream. It tells the instance what word we are looking for, input as a list of strings. These words can show up in any form, so it’s very broad.  If we didn’t put the hashtag in front of it, it would simply look for that word no matter where it showed up, so we might have too many results. By looking for hashtags, we narrow our search only to people using that specific hashtag, not just the word wherever it is. To search for terms, you have to put them into the list as a string before running the code. \n\nWhen the code is run by typing `python3 live_tweets.py` into the terminal, this is what the output looks like in the terminal.\n\n![Output in terminal](https://about.gitlab.com/images/blogimages/pythonwithpj5.png){: .shadow}\n\n\nThat’s it! That’s how the bot works, but we still need to talk about `config.py` and why we used it before. Here’s the contents of the file: \n\n```python\nimport os\nfrom dotenv import load_dotenv\n\nload_dotenv()\nconsumer_key = os.getenv(\"consumer_key\")\nconsumer_secret = os.getenv(\"consumer_secret\")\naccess_token = os.getenv(\"access_token\")\naccess_token_secret = os.getenv(\"access_token_secret\")\n```\n\nI tricked you! This doesn’t have the keys there either! Using `import os` and `import dotenv import load_dotenv` gives us access to something very important to keep secret keys safe: environmental variables. An environmental variable can be set in many different places, but in this case, our local repo has a file called `.env` that holds the actual keys.\n\nThis is there so I can test the app and run it on my machine. To use it somewhere else, you’d have to have environmental variables set up to hold the keys for the Twitter API. When I run my bots on Heroku, I keep the keys in the settings so it has access to the keys it needs to run. I use a `.gitignore` file that keeps my `.env` file from being committed to GitLab. \n\nAs you can see, the variables in `config.py` are set to `os.getenv(“name_of_key”)`. When we import `config.py` as `import config`, we gain access to these variables by calling `config.name_of_variable` in our main file. \n\nSo, for now, that’s what I built! It’s not much and I pieced it together using a lot of documentation from Twitter and Tweepy as well as a few tutorials and plenty of Stackoverflow, but it got built and it works the way I want it to!\n\nI’ve really enjoyed learning Python online and writing about it for everyone who has been reading it. I encourage anyone learning a new language or skill to write about it; it has really helped solidify my learning, and who knows, maybe I’ve helped someone else understand something in Python as well. \n\n",[1121,9,3247],{"slug":3285,"featured":6,"template":700},"learn-python-with-pj-part-5-building-something-with-the-twitter-api","content:en-us:blog:learn-python-with-pj-part-5-building-something-with-the-twitter-api.yml","Learn Python With Pj Part 5 Building Something With The Twitter Api","en-us/blog/learn-python-with-pj-part-5-building-something-with-the-twitter-api.yml","en-us/blog/learn-python-with-pj-part-5-building-something-with-the-twitter-api",{"_path":3291,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3292,"content":3298,"config":3303,"_id":3305,"_type":13,"title":3306,"_source":15,"_file":3307,"_stem":3308,"_extension":18},"/en-us/blog/learning-python-with-a-little-help-from-ai-code-suggestions",{"title":3293,"description":3294,"ogTitle":3293,"ogDescription":3294,"noIndex":6,"ogImage":3295,"ogUrl":3296,"ogSiteName":685,"ogType":686,"canonicalUrls":3296,"schema":3297},"Learning Python with a little help from AI","Use this guided tutorial, along with GitLab Duo Code Suggestions, to learn a new programming language.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663918/Blog/Hero%20Images/aipower.jpg","https://about.gitlab.com/blog/learning-python-with-a-little-help-from-ai-code-suggestions","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learning Python with a little help from AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2023-11-09\",\n      }",{"title":3293,"description":3294,"authors":3299,"heroImage":3295,"date":3300,"body":3301,"category":718,"tags":3302},[1646],"2023-11-09","Learning a new programming language can help broaden your software\ndevelopment expertise, open career opportunities, or create fun challenges.\nHowever, it can be difficult to decide on one specific approach to learning\na new language. Artificial intelligence (AI) can help. In this tutorial,\nyou'll learn how to leverage AI-powered GitLab Duo Code Suggestions for a\nguided experience in learning the Python programming language with a\npractical hands-on example.\n\n\n- [Preparations](#preparations)\n  - [VS Code](#vs-code)\n  - [Code Suggestions](#code-suggestions)\n- [Learning a new programming language:\nPython](#learning-a-new-programming-language-python)\n    - [Development environment for Python](#development-environment-for-python)\n    - [Hello, World](#hello-world)\n- [Start learning Python with a practical\nexample](#start-learning-python-with-a-practical-example)\n    - [Define variables and print them](#define-variables-and-print-them)\n    - [Explore variable types](#explore-variable-types)\n- [File I/O: Read and print a log file](#file-io-read-and-print-a-log-file)\n\n- [Flow control](#flow-control)\n    - [Loops and lists to collect files](#loops-and-lists-to-collect-files)\n    - [Conditionally collect files](#conditionally-collect-files)\n- [Functions](#functions)\n    - [Start with a simple log format](#start-with-a-simple-log-format)\n    - [String and data structure operations](#string-and-data-structure-operations)\n    - [Parse log files using regular expressions](#parse-log-files-using-regular-expressions)\n    - [Advanced log format: auth.log](#advanced-log-format-authlog)\n    - [Parsing more types: Structured logging](#parsing-more-types-structured-logging)\n- [Printing results and formatting](#printing-results-and-formatting)\n\n- [Dependency management and continuous\nverification](#dependency-management-and-continuous-verification)\n    - [Pip and pyenv: Bringing structure into Python](#pip-and-pyenv-bringing-structure-into-python)\n    - [Automation: Configure CI/CD pipeline for Python](#automation-configure-cicd-pipeline-for-python)\n- [What is next](#what-is-next)\n    - [Async learning exercises](#async-learning-exercises)\n    - [Share your feedback](#share-your-feedback)\n\n## Preparations \n\n\nChoose your [preferred and supported\nIDE](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-in-other-ides-and-editors),\nand follow the documentation to enable Code Suggestions for [GitLab.com\nSaaS](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-gitlab-saas)\nor [GitLab self-managed\ninstances](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-self-managed-gitlab).\n\n\nProgramming languages can require installing the language interpreter\ncommand-line tools or compilers that generate binaries from source code to\nbuild and run the application.\n\n\n**Tip:** You can also use [GitLab Remote Development\nworkspaces](/blog/quick-start-guide-for-gitlab-workspaces/) to\ncreate your own cloud development environments, instead of local development\nenvironments. This blog post focuses on using VS Code and the GitLab Web\nIDE. \n\n\n### VS Code\n\n\n[Install VS Code](https://code.visualstudio.com/download) on your client,\nand open it. Navigate to the `Extensions` menu and search for `gitlab\nworkflow`. Install the [GitLab Workflow extension for VS\nCode](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow).\nVS Code will also detect the programming languages, and offer to install\nadditional plugins for syntax highlighting and development experience. For\nexample, install the [Python\nextension](https://marketplace.visualstudio.com/items?itemName=ms-python.python).\n\n\n### Code Suggestions\n\n\nFamiliarize yourself with suggestions before actually verifying the\nsuggestions. GitLab Duo Code Suggestions are provided as you type, so you do\nnot need use specific keyboard shortcuts. To accept a code suggestion, press\nthe `tab` key. Also note that writing new code works more reliably than\nrefactoring existing code. AI is non-deterministic, which means that the\nsame suggestion may not be repeated after deleting the code suggestion.\nWhile Code Suggestions is in Beta, we are working on improving the accuracy\nof generated content overall. Please review the [known\nlimitations](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#known-limitations),\nas this could affect your learning experience.\n\n\n**Tip:** The latest release of Code Suggestions supports multiline\ninstructions. You can refine the specifications to your needs to get better\nsuggestions. We will practice this method throughout the blog post.\n\n\n## Learning a new programming language: Python  \n\n\nNow, let's dig into learning Python, which is one of the [supported\nlanguages in Code\nSuggestions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#supported-languages). \n\n\nBefore diving into the source code, make sure to set up your development\nenvironment.\n\n\n### Development environment for Python \n\n\n1) Create a new project `learn-python-ai` in GitLab, and clone the project\ninto your development environment. All code snippets are available in this\n[\"Learn Python with AI\"\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-python-ai).\n\n\n```shell\n\ngit clone https://gitlab.com/NAMESPACE/learn-python-ai.git\n\n\ncd learn-python-ai\n\n\ngit status\n\n```\n\n\n2) Install Python and the build toolchain. Example on macOS using Homebrew:\n\n\n```\n\nbrew install python\n\n```\n\n\n3) Consider adding a `.gitignore` file for Python, for example this\n[.gitignore template for\nPython](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/gitignore/Python.gitignore?ref_type=heads). \n\n\nYou are all set to learn Python! \n\n\n### Hello, World\n\n\nStart your learning journey in the [official\ndocumentation](https://www.python.org/about/gettingstarted/), and review the\nlinked resources, for example, the [Python\ntutorial](https://docs.python.org/3/tutorial/index.html). The\n[library](https://docs.python.org/3/library/index.html) and [language\nreference](https://docs.python.org/3/reference/index.html) documentation can\nbe helpful, too. \n\n\n**Tip:** When I touched base with Python in 2005, I did not have many use\ncases except as a framework to test Windows 2000 drivers. Later, in 2016, I\nrefreshed my knowledge with the book \"Head First Python, 2nd Edition,\"\nproviding great practical examples for the best learning experience – two\nweeks later, I could explain the differences between Python 2 and 3. You do\nnot need to worry about Python 2 – it has been deprecated some years ago,\nand we will focus only on Python 3 in this blog post. In August 2023, \"[Head\nFirst Python, 3rd\nEdition](https://www.oreilly.com/library/view/head-first-python/9781492051282/)\"\nwas published. The book provides a great learning resource, along with the\nexercises shared in this blog post. \n\n\nCreate a new file `hello.py` in the root directory of the project and start\nwith a comment saying `# Hello world`. Review and accept the suggestion by\npressing the `tab` key and save the file (keyboard shortcut: cmd s). \n\n\n```\n\n# Hello world\n\n```\n\n\nCommit the change to the Git repository. In VS Code, use the keyboard\nshortcut `ctrl shift G`, add a commit message, and hit `cmd enter` to\nsubmit. \n\n\nUse the command palette (`cmd shift p`) and search for `create terminal` to\nopen a new terminal. Run the code with the Python interpreter. On macOS, the\nbinary from Homebrew is called `python3`, other operating systems and\ndistributions might use `python` without the version.\n\n\n```shell\n\npython3 hello.py\n\n```\n\n\n![Hello World, hello GitLab Duo Code\nSuggestions](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_python_code_sugestions_hello_world.png)\n\n\n**Tip:** Adding code comments in Python starting with the `#` character\nbefore you start writing a function or algorithm will help Code Suggestions\nwith more context to provide better suggestions. In the example above, we\ndid that with `# Hello world`, and will continue doing so in the next\nexercises.\n\n\nAdd `hello.py` to Git, commit all changes and push them to your GitLab\nproject.\n\n\n```shell\n\ngit add hello.py\n\n\ngit commit -avm \"Initialize Python\"\n\n\ngit push\n\n```\n\n\nThe source code for all exercises in this blog post is available in this\n[\"Learn Python with AI\"\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-python-ai).\n\n\n## Start learning Python with a practical example \n\n\nThe learning goal in the following sections involves diving into the\nlanguage datatypes, variables, flow control, and functions. We will also\nlook into file operations, string parsing, and data structure operations for\nprinting the results. The exercises will help build a command-line\napplication that reads different log formats, works with the data, and\nprovides a summary. This will be the foundation for future projects that\nfetch logs from REST APIs, and inspire more ideas such as rendering images,\ncreating a web server, or adding Observability metrics.\n\n\n![Parsing log files into structured objects, example result after following\nthe\nexercises](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_terminal_parsing_logs_and_pretty_print_results.png)\n\n\nAs an experienced admin, you can put the script into production and use\nreal-world log format exmples. Parsing and analyzing logs in stressful\nproduction incidents can be time-consuming. A local CLI tool is sometimes\nfaster than a log management tool.\n\n\nLet's get started: Create a new file called `log_reader.py` in the directory\nroot, add it to Git, and create a Git commit.\n\n\n### Define variables and print them\n\n\nAs a first step, we need to define the log files location, and the expected\nfile suffix. Therefore, let's create two variables and print them. Actually,\nask Code Suggestions to do that for you by writing only the code comments\nand accepting the suggestions. Sometimes, you need to experiment with\nsuggestions and delete already accepted code blocks. Do not worry – the\nquality of the suggestions will improve over time as the model generates\nbetter suggestions with more context.\n\n\n![Define log path and file suffix\nvariables](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_log_reader_variables_01.png){:\n.shadow}\n\n\n![Print the variables to\nverify](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_log_reader_variables_02.png){:\n.shadow}\n\n\n```python\n\n# Specify the path and file suffix in variables\n\npath = '/var/log/'\n\nfile_suffix = '.log'\n\n\n# Print the variables \n\n\nprint(path)\n\nprint(file_suffix)\n\n```\n\n\nNavigate into the VS Code terminal and run the Python script:\n\n\n```shell\n\npython3 log_reader.py\n\n```\n\n\n![VS Code terminal, printing the\nvariables](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_terminal_print_variables.png)\n\n\nPython supports many different types in the [standard\nlibrary](https://docs.python.org/3/library/index.html). Most common types\nare: Numeric (int, float, complex), Boolean (True, False), and String (str).\nData structures include support for lists, tuples, and dictionaries. \n\n\n### Explore variable types \n\n\nTo practice different variable types, let's define a limit of log files to\nread as a variable with the `integer` type.\n\n\n![Log file\nvariable](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_log_reader_variables_03.png){:\n.shadow}\n\n\n```python\n\n# Define log file limit variable \n\nlog_file_limit = 1024 \n\n```\n\n\nCreate a Boolean variable that forces to read all files in the directory, no\nmatter the log file suffix. \n\n\n```python\n\n# Define boolean variable whether to read all files recursively\n\nread_all_files_recursively = True\n\n```\n\n\n## File I/O: Read and print a log file\n\n\nCreate a directory called `log-data` in your project tree. You can copy all\nfile examples from the [log-data directory in the example\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-python-ai/-/tree/main/log-data?ref_type=heads).  \n\n\nCreate a new file `sample.log` with the following content, or any other two\nlines that provide a different message at the end.\n\n\n```\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: dpkg-db-backup.service: Deactivated\nsuccessfully.\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: Finished Daily dpkg database backup\nservice.\n\n```\n\n\nInstruct Code Suggestions to read the file `log-data/sample.log` and print\nthe content. \n\n\n![Code Suggestions: Read log file and print\nit](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_read_log_file_and_print.png){:\n.shadow}\n\n\n```python\n\n# Read the file in log-data/sample.log and print its content\n\nwith open('log-data/sample.log', 'r') as f:\n    print(f.read())\n```\n\n\n**Tip:** You will notice the indent here. The `with open() as f:` statement\nopens a new scope where `f` is available as stream. This flow requires\nindenting )`tab`) the code block, and perform actions in this scope, calling\n`f.read()` to read the file contents, and passing the immediate value as\nparameter into the `print()` function.\n\n\nNavigate into the terminal, and run the script again with `python3\nlog_reader.py`. You will see the file content shown in the VS Code editor,\nalso printed into the terminal.\n\n\n![VS Code terminal: Read log file, and print\nit](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_terminal_print_logfile_show_file_sample.png)\n\n\n## Flow control \n\n\nReading one log file is not enough – we want to analyze all files in a given\ndirectory recursively. For the next exercise, we instruct Code Suggestions\nto create an index of all files. \n\n\nPrepare the `log-data` directory with more example files from the [log-data\ndirectory in the example\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-python-ai/-/tree/main/log-data?ref_type=heads).\nThe directory tree should look as follows:\n\n\n```shell\n\ntree log-data                                                             ─╯\n\nlog-data\n\n├── sample.log\n\n└── var\n    └── log\n        ├── auth.log\n        ├── syslog.log\n        └── syslog_structured.log\n\n3 directories, 4 files\n\n```\n\n\n### Loops and lists to collect files \n\n\nModify the `path` variable to use the value `log-data/`. \n\n\n```python\n\n# Specify the path and file suffix in variables\n\npath = 'log-data/'\n\nfile_suffix = '.log'\n\n```\n\n\nTell Code Suggestions to read all file paths in the directory into a list.\nAfter the collection loop, print the list of file paths. \n\n\n```python\n\n# Read all file paths in the directory into a list\n\n\n# Print the list of log file paths\n\n```\n\n\n![Code Suggestion, collect file\npaths](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_collect_files.png){:\n.shadow}\n\n\nAn example of a suggestion could look like this:\n\n\n```python\n\n# Read all file paths in the directory into a list\n\nimport os\n\n\n# Read all file paths in the directory into a list\n\nlog_files = []\n\nfor root, directories, files in os.walk(path):\n    for file in files:\n        if file.endswith(file_suffix):\n            log_files.append(os.path.join(root, file))\n\n# Print the list of log file paths\n\n\nprint(log_files)\n\n```\n\n\nLet's analyze what happens here: The `import os` statement is required to\nbring the `os` library into the current scope, and we are able to call\n`os.walk()` later. The `log_files`\n[list](https://docs.python.org/3/library/stdtypes.html#sequence-types-list-tuple-range)\nis initialized as an empty list using empty brackets. `os.walk()` expects a\npath as directory tree to start searching for files. The function returns a\ngenerator object, which will walk the directory tree recursively. The\nreturned tuples include the collected files, which we want to access in a\nloop iteraor, using the `for` loop notation. When the file ends with the\ndefined suffix, its full path is appended to the list. This is an optimized\nand complex example. Do not worry about understanding everything just yet -\nwe will revisit this later. \n\n\n### Conditionally collect files \n\n\nModify the instructions and add the Boolean flag evaluation for\n`read_all_files_recursively` as an input to the glob() function. \n\n\n```python\n\n# Read all file paths in the top level directory into a list\n\n# If read_all_files_recursively is True, read all files recursively\n\n```\n\n\n![Code Suggestions, read files recursively\nconditionally](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_read_files_recursively_conditionally.png){:\n.shadow}\n\n\n```\n\n# Read all file paths in the top level directory \n\nimport os\n\n\n# Read all file paths in the top level directory into a list\n\n# If read_all_files_recursively is True, read all files recursively\n\n\nlog_files = []\n\nfor file in os.listdir(path):\n    if file.endswith(file_suffix):\n        log_files.append(os.path.join(path, file))\n\nif read_all_files_recursively:\n    for root, directories, files in os.walk(path):\n        for file in files:\n            if file.endswith(file_suffix):               \n                log_files.append(os.path.join(root, file))\n\n# Print log_files\n\nprint(log_files)\n\n```\n\n\nThe result is not optimal yet because it always executes the first loop, and\noptionally the second loop. This flow leads to duplicated results when the\nscript is executed.\n\n\n```\n\npython3 log_reader.py\n\n\n['log-data/sample.log', 'log-data/sample.log', 'log-data/var/log/auth.log']\n\n```\n\n\nExperiment with Code Suggestions instructions to get a solution for the\nproblem. There are different approaches you can take: \n\n\n1) A potential solution is to wrap the source code into an if-then-else\nblock, and move the `os.listdir()` loop into the else-block. \n\n\n```python\n\nif read_all_files_recursively:\n    for root, directories, files in os.walk(path):\n        for file in files:\n            if file.endswith(file_suffix):               \n                log_files.append(os.path.join(root, file))\nelse:\n    for file in os.listdir(path):\n        if file.endswith(file_suffix):\n            log_files.append(os.path.join(path, file))  \n\n```\n\n\n2) Alternatively, do not use `append()` to always add a new list entry, but\ncheck if the item exists in the list first. \n\n\n```python\n\nfor file in os.listdir(path):\n    if file.endswith(file_suffix):\n        # check if the entry exists in the list already\n        if os.path.isfile(os.path.join(path, file)):\n            log_files.append(os.path.join(path, file))\n\nif read_all_files_recursively:\n    for root, directories, files in os.walk(path):\n        for file in files:\n            if file.endswith(file_suffix):\n                # check if the entry exists in the list already\n                if file not in log_files:\n                    log_files.append(os.path.join(root, file))\n```\n\n\n3) Or, we could eliminate duplicate entries after collecting all items.\nPython allows converting lists into\n[sets](https://docs.python.org/3/library/stdtypes.html#set-types-set-frozenset),\nwhich hold unique entries. After applying `set()`, you can again convert the\nset back into a list. Code Suggestions knows about this possibility, and\nwill help with the comment `# Ensure that only unique file paths are in the\nlist` \n\n\n![Code Suggestions, converting a list to unique\nitems](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_unique_list.png){:\n.shadow}\n\n\n```python\n\n# Ensure that only unique file paths are in the list\n\n\nlog_files = list(set(log_files))\n\n```\n\n\n4) Take a step back and evaluate whether the variable\nread_all_files_recursively makes sense. Maybe the default behavior should\njust be reading all files recursively?\n\n\n**Tip for testing different paths in VS Code:** Select the code blocks, and\npress [`cmd /` on\nmacOS](https://code.visualstudio.com/docs/getstarted/keybindings) to comment\nout the code. \n\n\n## Functions \n\n\nLet's create a function called `parse_log_file` that parses a log file, and\nreturns the extracted data. We will define the expected log format and\ncolumns to extract, following the [syslog format\nspecification](https://en.wikipedia.org/wiki/Syslog). There are different\nlog format types and also customized formats by developers that need to be\ntaken into account – exercise for later. \n\n\n### Start with a simple log format \n\n\nInspect a running Linux VM, or use the following example log file example\nfor additional implementation.\n\n\n```\n\nless /var/log/syslog | grep -v docker \n\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: Starting Daily dpkg database backup\nservice...\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: Starting Rotate log files...\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: dpkg-db-backup.service: Deactivated\nsuccessfully.\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: Finished Daily dpkg database backup\nservice.\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: logrotate.service: Deactivated\nsuccessfully.\n\nOct 17 00:00:04 ebpf-chaos systemd[1]: Finished Rotate log files.\n\nOct 17 00:17:01 ebpf-chaos CRON[727495]: (root) CMD (   cd / && run-parts\n--report /etc/cron.hourly)\n\n```\n\n\nWe can create an algorithm to split each log line by whitespaces, and then\njoin the results again. Let's ask Code Suggestions for help. \n\n\n```python\n\n# Split log line \"Oct 17 00:00:04 ebpf-chaos systemd[1]: Finished Rotate log\nfiles.\" by whitespaces and save in a list\n\n\nlog_line = \"Oct 17 00:00:04 ebpf-chaos systemd[1]: Finished Rotate log\nfiles.\"\n\nlog_line_split = log_line.split(\" \")\n\nprint(log_line_split)\n\n```\n\n\nRun the script again to verify the result.\n\n\n```shell\n\npython3 log_reader.py\n\n\n['Oct', '17', '00:00:04', 'ebpf-chaos', 'systemd[1]:', 'Finished', 'Rotate',\n'log', 'files.']\n\n```\n\n\nThe first three items are part of the datetime string, followed by the host,\nservice, and remaining log message items. Let's practice string operations\nin Python as the next step. \n\n\n### String and data structure operations\n\n\nLet's ask Code Suggestions for help with learning to join strings, and\nperform list operations.\n\n\n1. Join the first three items with a whitespace again. \n\n2. Keep host and service. \n\n3. Join the remaining variable item count into a string, separated with\nwhitespaces, again. \n\n4. Store the identified column keys, and their respective values in a new\ndata structure:\n[dictionary](https://docs.python.org/3/library/stdtypes.html#mapping-types-dict). \n\n\n![Code suggestions for list items with string\noperations](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_list_items_string_join_extract.png){:\n.shadow}\n\n\n```shell \n\npython3 log_reader.py\n\n\n# Array\n\n['Oct', '17', '00:00:04', 'ebpf-chaos', 'systemd[1]:', 'Finished', 'Rotate',\n'log', 'files.']\n\n\n# Dictionary \n\n{'datetime': 'Oct 17 00:00:04', 'host': 'ebpf-chaos', 'service':\n'systemd[1]:', 'message': ' ebpf-chaos systemd[1]: Finished Rotate log\nfiles.'}\n\n```\n\n\nA working suggestion can look like the following:\n\n\n```python\n\n# Initialize results dictionary with empty values for datetime, host,\nservice, message\n\n# Loop over log line split \n\n# Join the first three list items as date string\n\n# Item 4: host \n\n# Item 5: service\n\n# Join the remaining items into a string, separated with whitespaces \n\n# Print the results after the loop \n\n\nresults = {'datetime': '', 'host': '', 'service': '', 'message': ''}\n\n\nfor item in log_line_split:\n\n    if results['datetime'] == '':\n        results['datetime'] = ' '.join(log_line_split[0:3])\n\n    elif results['host'] == '':\n        results['host'] = log_line_split[3]\n\n    elif results['service'] == '':\n        results['service'] = log_line_split[4]\n\n    else:\n        results['message'] += ' ' + item\n\nprint(results)\n\n\n```\n\n\nThe suggested algorithm loops over all log line items, and applies the same\noperation for the first three items. `log_line_split[0:3]` extracts a slice\nof three items into a new list. Calling `join()` on a separator character\nand passing the array as an argument joins the items into a string. The\nalgorithm continues to check for not initialized values for host (Item 4)\nand service (Item 5)and concludes with the remaining list items appended\ninto the message string. To be honest, I would have used a slightly\ndifferent algorithm, but it is a great learning curve to see other\nalgorithms, and ways to implement them. Practice with different\ninstructions, and data structures, and continue printing the data sets. \n\n\n**Tip:** If you need to terminate a script early, you can use `sys.exit()`.\nThe remaining code will not be executed. \n\n\n```python\n\nimport sys \n\nsys.exit(1)\n\n```\n\n\nImagine doing these operations for different log formats, and message types\n– it can get complicated and error-prone very quickly. Maybe there is\nanother approach. \n\n\n### Parse log files using regular expressions\n\n\nThere are different syslog format RFCs – [RFC\n3164](https://datatracker.ietf.org/doc/html/rfc3164) is obsolete but still\nfound in the wild as default configuration (matching the pattern above),\nwhile [RFC 5424](https://datatracker.ietf.org/doc/html/rfc5424) is more\nmodern, including datetime with timezone information. Parsing this format\ncan be tricky, so let's ask Code Suggestions for advice. \n\n\nIn some cases, the suggestions include regular expressions. They might not\nmatch immediately, making the code more complex to debug, with trial and\nerrors. A good standalone resource to text and explain regular expressions\nis [regex101.com](https://regex101.com/).  \n\n\n**Tip:** You can skip diving deep into regular expressions using the\nfollowing code snippet as a quick cheat. The next step involves instructing\nCode Suggestions to use these log patterns, and help us extract all valuable\ncolumns. \n\n\n```python\n\n# Define the syslog log format regex in a dictionary\n\n# Add entries for RFC3164, RFC5424\n\nregex_log_pattern = {\n    'rfc3164': '([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})\\s([\\w][\\w\\d\\.@-]*)\\s(.*)$',\n    'rfc5424': '(?:(\\d{4}[-]\\d{2}[-]\\d{2}[T]\\d{2}[:]\\d{2}[:]\\d{2}(?:\\.\\d{1,6})?(?:[+-]\\d{2}[:]\\d{2}|Z)?)|-)\\s(?:([\\w][\\w\\d\\.@-]*)|-)\\s(.*)$;'\n}\n\n```\n\n\nWe know what the function should do, and its input parameters – the file\nname, and a log pattern to match. The log lines should be split by this\nregular expression, returning a key-value dictionary for each log line. The\nfunction should return a list of dictionaries. \n\n\n```python\n\n# Create a function that parses a log file\n\n# Input parameter: file path\n\n# Match log line against regex_log_pattern\n\n# Return the results as dictionary list: log line, pattern, extracted\ncolumns\n\n```\n\n\n![Code suggestion based on a multiline comment instruction to get a function\nthat parses a log file based on regex\npatterns](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_log_format_regex_function_instructions_01.png){:\n.shadow}\n\n\nRemember the indent for opening a new scope? The same applies for functions\nin Python. The `def` identifier requires a function name, and a list of\nparameters, followed by an opening colon. The next lines of code require the\nindent. VS Code will help with live-linting wrong indent, before the script\nexecution fails, or the CI/CD pipelines. \n\n\nContinue with Code Suggestions – it might already know that you want to\nparse all log files, and parse them using the newly created function. \n\n\n![Code suggestion to parse all log files, and print the result\nset](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_log_format_regex_function_instructions_02.png){:\n.shadow}\n\n\nA full working example can look like this: \n\n\n```\n\nimport os\n\n\n# Specify the path and file suffix in variables\n\npath = 'log-data/'\n\nfile_suffix = '.log'\n\n\n# Read all file paths in the directory into a list\n\nlog_files = []\n\nfor root, directories, files in os.walk(path):\n    for file in files:\n        if file.endswith(file_suffix):\n            log_files.append(os.path.join(root, file))\n\n# Define the syslog log format regex in a dictionary\n\n# Add entries for RFC3164, RFC5424\n\nregex_log_pattern = {\n    'rfc3164': '([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})\\s([\\w][\\w\\d\\.@-]*)\\s(.*)$',\n    'rfc5424': '(?:(\\d{4}[-]\\d{2}[-]\\d{2}[T]\\d{2}[:]\\d{2}[:]\\d{2}(?:\\.\\d{1,6})?(?:[+-]\\d{2}[:]\\d{2}|Z)?)|-)\\s(?:([\\w][\\w\\d\\.@-]*)|-)\\s(.*)$;'\n}\n\n\n# Create a function that parses a log file\n\n# Input parameter: file path\n\n# Match log line against regex_log_pattern\n\n# Return the results as dictionary list: log line, pattern name, extracted\ncolumns\n\nimport re\n\n\ndef parse_log_file(file_path):\n    # Read the log file\n    with open(file_path, 'r') as f:\n        log_lines = f.readlines()\n\n    # Create a list to store the results\n    results = []\n\n    # Iterate over the log lines\n    for log_line in log_lines:\n        # Match the log line against the regex pattern\n        for pattern_name, pattern in regex_log_pattern.items():\n            match = re.match(pattern, log_line)\n\n            # If the log line matches the pattern, add the results to the list\n            if match:\n                extracted_columns = match.groups()\n                results.append({\n                    'log_line': log_line,\n                    'pattern_name': pattern_name,\n                    'extracted_columns': extracted_columns,\n                    'source_file': file_path\n                })\n\n    # Return the results\n    return results\n\n# Parse all files and print results\n\nfor log_file in log_files:\n    results = parse_log_file(log_file)\n    print(results)\n```\n\n\nLet's unpack what the `parse_log_file()` function does:\n\n\n1. Opens the file from `file_path` parameter. \n\n2. Reads all lines into a new variable `log_lines`. \n\n3. Creates a results list to store all items. \n\n4. Iterates over the log lines. \n\n5. Matches against all regex patterns configured in regex_log_pattern. \n\n6. If a match is found, extracts the matching column values.\n\n7. Creates a results item, including the values for the keys `log_line`,\n`pattern_name`, `extracted_colums`, `source_file`. \n\n8. Appends the results item to the results list.\n\n9. Returns the results list. \n\n\nThere are different variations to this – especially for the returned result\ndata structure. For this specific case, log lines come as list already.\nAdding a dictionary object instead of a raw log line allows function callers\nto extract the desired information in the next step. Once a working example\nhas been implemented, you can refactor the code later, too. \n\n\n### Advanced log format: auth.log\n\n\nParsing the syslog on a Linux distribution might not unveil the necessary\ndata to analyze. On a virtual machine that exposes port 22 (SSH) to the\nworld, the authentication log is much more interesting – plenty of bots and\nmalicious actors testing default password combinations and often brute force\nattacks.\n\n\nThe following snippet from `/var/log/auth.log` on one of my private servers\nshows the authentication log format and the random attempts from bots using\ndifferent usernames, etc. \n\n\n```\n\nOct 15 00:00:19 ebpf-chaos sshd[3967944]: Failed password for invalid user\nubuntu from 93.254.246.194 port 48840 ssh2\n\nOct 15 00:00:20 ebpf-chaos sshd[3967916]: Failed password for root from\n180.101.88.227 port 44397 ssh2\n\nOct 15 00:00:21 ebpf-chaos sshd[3967944]: Received disconnect from\n93.254.246.194 port 48840:11: Bye Bye [preauth]\n\nOct 15 00:00:21 ebpf-chaos sshd[3967944]: Disconnected from invalid user\nubuntu 93.254.246.194 port 48840 [preauth]\n\nOct 15 00:00:24 ebpf-chaos sshd[3967916]: Failed password for root from\n180.101.88.227 port 44397 ssh2\n\nOct 15 00:00:25 ebpf-chaos sshd[3967916]: Received disconnect from\n180.101.88.227 port 44397:11:  [preauth]\n\nOct 15 00:00:25 ebpf-chaos sshd[3967916]: Disconnected from authenticating\nuser root 180.101.88.227 port 44397 [preauth]\n\nOct 15 00:00:25 ebpf-chaos sshd[3967916]: PAM 2 more authentication\nfailures; logname= uid=0 euid=0 tty=ssh ruser= rhost=180.101.88.227 \nuser=root\n\nOct 15 00:00:25 ebpf-chaos sshd[3967998]: Invalid user teamspeak from\n185.218.20.10 port 33436\n\n```\n\n\n**Tip for intrusion prevention:** Add a firewall setup, and use\n[fail2ban](https://en.wikipedia.org/wiki/Fail2ban) to block invalid auth\nlogins. \n\n\nThe next exercise is to extend the logic to understand the free form log\nmessage parts, for example `Failed password for invalid user ubuntu from\n93.254.246.194 port 48840 ssh2`. The task is to store the data in an\noptional dictionary with key value pairs. \n\n\nCreate a new function that takes the previously parsed log line results as\ninput, and specifically parses the last list item for each line.\n\n\n1. Count the number of `Failed password` and `Invalid user` messages.\n\n2. Return the results with count, log file, pattern \n\n\n![Code suggestions for a log file message parser to count auth.log\nfailures](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_parse_log_message_auth_log.png){:\n.shadow}\n\n\nA working suggestion can look like the following code:\n\n\n```python\n\n# Create a function that parses a log file message from the last\nextracted_columns entry \n\n# Input: Parsed log lines results list \n\n# Loop over all log lines in the list, and extract the last list item as\nmessage \n\n# Count failure strings in the message: Failed password, Invalid user \n\n# Return the results if failure count greater 0: log_file, count, failure\nstring\n\ndef parse_log_file_message(results):\n    failure_results = []\n\n    # Iterate over the log lines\n    for result in results:\n        # Extract the message from the last list item\n        message = result['extracted_columns'][-1]\n\n        # Count the number of failure strings in the message\n        failure_count = message.count('Failed password') + message.count('Invalid user')\n\n        # If the failure count is greater than 0, add the results to the list\n        if failure_count > 0:\n            failure_results.append({\n                'log_file': result['source_file'],\n                'count': failure_count,\n                'failure_string': message\n            })\n\n    # Return the results\n    return failure_results\n\n# Parse all files and print results\n\nfor log_file in log_files:\n    results = parse_log_file(log_file)\n    failure_results = parse_log_file_message(results)\n    print(failure_results)\n```\n\n\nThe algorithm follows the previous implementations: First, create a results\narray to store matching data. Then, iterate over the already parsed\nlog_lines in the list. Each log line contains the `extracted_columns` key,\nwhich holds the free-form message string at the end. The next step is to\ncall the string object function `count()` to count how many times a given\ncharacter sequence is contained in a string. The returned numbers are added\nup to the `failure_count` variable. If it is greater than zero, the result\nis added to the results list, including the `log_file`, `count` and\n`failure_string` key-value pairs. After returning the parsed log message\nresults, loop through all log files, parse them, and print the results\nagain. \n\n\nExecute the script to inspect the detected matches. Note that the data\nstructure can be optimized in future learning steps.\n\n\n```\n\npython3 log_reader.py\n\n\n[{'log_file': 'log-data/var/log/auth.log', 'count': 1, 'failure_string':\n'sshd[3967944]: Failed password for invalid user ubuntu from 93.254.246.194\nport 48840 ssh2'}, {'log_file': 'log-data/var/log/auth.log', 'count': 1,\n'failure_string': 'sshd[3967916]: Failed password for root from\n180.101.88.227 port 44397 ssh2'}, {'log_file': 'log-data/var/log/auth.log',\n'count': 1, 'failure_string': 'sshd[3967916]: Failed password for root from\n180.101.88.227 port 44397 ssh2'}, {'log_file': 'log-data/var/log/auth.log',\n'count': 1, 'failure_string': 'sshd[3967998]: Invalid user teamspeak from\n185.218.20.10 port 33436'}, {'log_file': 'log-data/var/log/auth.log',\n'count': 1, 'failure_string': 'sshd[3967998]: Failed password for invalid\nuser teamspeak from 185.218.20.10 port 33436 ssh2'}, {'log_file':\n'log-data/var/log/auth.log', 'count': 1, 'failure_string': 'sshd[3968077]:\nInvalid user mcserver from 218.211.33.146 port 50950'}]\n\n\n```\n\n\n### Parsing more types: Structured logging\n\n\nApplication developers can use the structured logging format to help machine\nparsers to extract the key value pairs. Prometheus provides this information\nin the following structure in syslog:\n\n\n```\n\nOct 17 19:00:10 ebpf-chaos prometheus[594]: ts=2023-10-17T19:00:10.425Z\ncaller=compact.go:519 level=info component=tsdb m\n\nsg=\"write block\" mint=1697558404661 maxt=1697565600000\nulid=01HCZG4ZX51GTH8H7PVBYDF4N6 duration=148.675854ms\n\nOct 17 19:00:10 ebpf-chaos prometheus[594]: ts=2023-10-17T19:00:10.464Z\ncaller=head.go:1213 level=info component=tsdb msg\n\n=\"Head GC completed\" caller=truncateMemory duration=6.845245ms\n\nOct 17 19:00:10 ebpf-chaos prometheus[594]: ts=2023-10-17T19:00:10.467Z\ncaller=checkpoint.go:100 level=info component=tsd\n\nb msg=\"Creating checkpoint\" from_segment=2308 to_segment=2309\nmint=1697565600000\n\nOct 17 19:00:10 ebpf-chaos prometheus[594]: ts=2023-10-17T19:00:10.517Z\ncaller=head.go:1185 level=info component=tsdb msg\n\n=\"WAL checkpoint complete\" first=2308 last=2309 duration=50.052621ms\n\n```\n\n\nThis format is easier to parse for scripts, because the message part can be\nsplit by whitespaces, and the assignment character `=`. Strings that contain\nwhitespaces are guaranteed to be enclosed with quotes. The downside is that\nnot all programming language libraries provide ready-to-use structured\nlogging libraries, making it harder for developers to adopt this format. \n\n\nPractice following the previous example to parse the `auth.log` format with\nadditional information. Tell Code Suggestions that you are expecting\nstructured logging format with key-value pairs, and which returned data\nstructure would be great:\n\n\n```python\n\n# Create a function that parses a log file message from the last\nextracted_columns entry \n\n# Input: Parsed log lines results list \n\n# Loop over all log lines in the list, and extract the last list item as\nmessage \n\n# Parse structured logging key-value pairs into a dictionary\n\n# Return results: log_file, dictionary \n\n```\n\n\n![Code suggestions for parsing structured logging format in the log file\nmessage\npart](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_propose_structured_logging_message_parser.png){:\n.shadow}\n\n\n### Printing results and formatting\n\n\nMany of the examples used the `print()` statement to print the content on\nthe terminal. Python objects in the standard library support text\nrepresentation, and for some types it makes more sense (string, numbers),\nothers cannot provide much details (functions, etc.). \n\n\nYou can also pretty-print almost any data structure (lists, sets,\ndictionaries) in Python. The JSON library can format data structures in a\nreadable format, and use a given spaces indent to draw the JSON structure on\nthe terminal. Note that we use the `import` statement here to bring\nlibraries into the current scope, and access their methods, for example\n`json.dumps`. \n\n\n```python\n\nimport json \n\nprint(json.dumps(structured_results, indent=4))\n\n```\n\n\n![Parsing log files into structured objects, example result after following\nthe\nexercises](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_terminal_parsing_logs_and_pretty_print_results.png)\n\n\nPractice with modifying the existing source code, and replace the code\nsnippets where appropriate. Alternatively, create a new function that\nimplements pretty printing.\n\n\n```python\n\n# Create a pretty print function with indent 4 \n\n```\n\n\n![Code suggestions for pretty-print\nfunction](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/vs_code_code_suggestions_pretty_print.png){:\n.shadow}\n\n\nThis idea works in a similar fashion with creating your own logger\nfunctions...but we have to stop learning and take a break. Before we\nconclude the first blog post in the learning series, let's ensure that CI/CD\nand dependencies are set up properly for future exercises and async\npractice. \n\n\n## Dependency management and continuous verification  \n\n\n### Pip and pyenv: Bringing structure into Python \n\n\nDependencies can be managed in the [`requirements.txt`\nfile](https://pip.pypa.io/en/stable/reference/requirements-file-format/),\nincluding optional version dependencies. Using `requirements.txt` file also\nhas the advantage of being the single source of truth for local development\nenvironments and running continuous builds with GitLab CI/CD. They can use\nthe same installation command:\n\n\n```shell\n\npip install -r requirements.txt\n\n```\n\n\nSome Linux distributions do not install the pip package manager by default,\nfor example, Ubuntu/Debian require to install the `python3-pip` package. \n\n\nYou can manage different virtual environments using\n[venv](https://docs.python.org/3/library/venv.html). This workflow can be\nbeneficial to install Python dependencies into the virtual environment,\ninstead of globally into the OS path which might break on upgrades. \n\n\n```shell\n\npip install virtualenv\n\nvirtualenv venv\n\nsource venv/bin/activate \n\n```\n\n\n### Automation: Configure CI/CD pipeline for Python\n\n\nThe [CI/CD pipeline](https://docs.gitlab.com/ee/ci/) should continuously\nlint, test, and build the code. You can mimic the steps from the local\ndevelopment, and add testing more environments and versions: \n\n\n1. Lint the source code and check for formatting errors. The example uses\n[Pyflakes](https://pypi.org/project/pyflakes/), a mature linter, and\n[Ruff](https://docs.astral.sh/ruff/ ), a fast linter written in Rust. \n\n2. Cache dependencies installed using the pip package manager, following the\ndocumentation for [Python caching in GitLab\nCI/CD](https://docs.gitlab.com/ee/ci/caching/#cache-python-dependencies).\nThis saves time and resources on repeated CI/CD pipeline runs.\n\n3. Use parallel matrix builds to test different Python versions, based on\nthe available container images on Docker Hub and their tags. \n\n\n```yaml\n\nstages:\n  - lint\n  - test\n\ndefault:\n  image: python:latest\n  cache:                      # Pip's cache doesn't store the python packages\n    paths:                    # https://pip.pypa.io/en/stable/topics/caching/\n      - .cache/pip\n  before_script:\n    - python -V               # Print out python version for debugging\n    - pip install virtualenv\n    - virtualenv venv\n    - source venv/bin/activate\n\nvariables:  # Change pip's cache directory to be inside the project\ndirectory since we can only cache local items.\n  PIP_CACHE_DIR: \"$CI_PROJECT_DIR/.cache/pip\"\n\n# lint template\n\n.lint-tmpl:\n  script:\n    - echo \"Linting Python version $VERSION\"\n  parallel:\n    matrix:\n      - VERSION: ['3.9', '3.10', '3.11', '3.12']   # https://hub.docker.com/_/python\n\n# Lint, using Pyflakes: https://pypi.org/project/pyflakes/ \n\nlint-pyflakes:\n  extends: [.lint-tmpl]\n  script:\n    - pip install -r requirements.txt\n    - find . -not -path './venv' -type f -name '*.py' -exec sh -c 'pyflakes {}' \\;\n\n# Lint, using Ruff (Rust): https://docs.astral.sh/ruff/ \n\nlint-ruff:\n  extends: [.lint-tmpl]\n  script:\n    - pip install -r requirements.txt\n    - ruff .\n```\n\n\n![GitLab CI/CD Python lint job view, part of matrix\nbuilds](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/gitlab_cicd_python_lint_job_log_view.png)\n\n\n## What is next \n\n\nFun fact: GitLab Duo Code Suggestions also helped writing this blog post in\nVS Code, knowing about the context. In the screenshot, I just wanted to add\na tip about [regex101](https://regex101.com/), and GitLab Duo already knew. \n\n\n![Writing the GitLab blog post in VS Code with support from GitLab Duo Code\nSuggestions](https://about.gitlab.com/images/blogimages/learn-python-with-ai-code-suggestions-getting-started/gitlab_duo_code_suggestions_helping_write_the_learning_python_ai_blog_post.png)\n\n\nIn an upcoming blog, we will look into advanced learning examples with more\npractical (log) filtering and parallel operations, how to fetch logs from\nAPI endpoints (CI/CD job logs for example), and more data analytics and\nobservability. Until then, here are a few recommendations for practicing\nasync.\n\n\n### Async learning exercises\n\n\n- Implement the missing `log_file_limit` variable check. \n\n- Print a summary of the results in Markdown, not only JSON format. \n\n- Extend the script to accept a search filter as environment variable.\nPrint/count only filtered results. \n\n- Extend the script to accept a date range. It might require parsing the\ndatetime column in a time object to compare the range. \n\n- Inspect a GitLab CI/CD pipeline job log, and download the raw format.\nExtend the log parser to parse this specific format, and print a summary. \n\n\n### Share your feedback\n\n\nWhich programming language are you learning or considering learning? Start a\nnew topic on our [community](/community/) forum or Discord and share your\nexperience.\n\n\nWhen you use [GitLab Duo](/gitlab-duo/) Code Suggestions, please share your\nthoughts and feedback [in the feedback\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/405152).\n",[495,9,956,720],{"slug":3304,"featured":6,"template":700},"learning-python-with-a-little-help-from-ai-code-suggestions","content:en-us:blog:learning-python-with-a-little-help-from-ai-code-suggestions.yml","Learning Python With A Little Help From Ai Code Suggestions","en-us/blog/learning-python-with-a-little-help-from-ai-code-suggestions.yml","en-us/blog/learning-python-with-a-little-help-from-ai-code-suggestions",{"_path":3310,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3311,"content":3315,"config":3320,"_id":3322,"_type":13,"title":3323,"_source":15,"_file":3324,"_stem":3325,"_extension":18},"/en-us/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started",{"title":3312,"description":3294,"ogTitle":3312,"ogDescription":3294,"noIndex":6,"ogImage":3295,"ogUrl":3313,"ogSiteName":685,"ogType":686,"canonicalUrls":3313,"schema":3314},"Learning Rust with a little help from AI","https://about.gitlab.com/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Learning Rust with a little help from AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2023-08-10\",\n      }",{"title":3312,"description":3294,"authors":3316,"heroImage":3295,"date":3317,"body":3318,"category":718,"tags":3319},[1646],"2023-08-10","Learning a new programming language can help broaden your software\ndevelopment expertise, open career opportunities, or create fun challenges.\nHowever, it can be difficult to decide on one specific approach to learning\na new language. Artificial intelligence (AI) can help. In this tutorial,\nyou'll learn how to leverage AI-powered GitLab Duo Code Suggestions for a\nguided experience in learning the Rust programming language.\n\n\n- [Preparations](#preparations)\n  - [VS Code](#vs-code)\n  - [Code Suggestions](#code-suggestions)\n- [Learning a new programming language:\nRust](#learning-a-new-programming-language-rust)\n    - [Development environment for Rust](#development-environment-for-rust)\n    - [Hello, World](#hello-world)\n- [Cargo: Bringing structure into Rust](#cargo-bringing-structure-into-rust)\n\n- [Automation: Configure CI/CD pipeline for\nRust](#automation-configure-cicd-pipeline-for-rust)\n\n- [Continue learning Rust](#continue-learning-rust)\n    - [Define variables and print them](#define-variables-and-print-them)\n    - [Explore variable types](#explore-variable-types)\n    - [Flow control: Conditions and loops](#flow-control-conditions-and-loops)\n    - [Functions](#functions)\n    - [Testing](#testing)\n- [What is next](#what-is-next)\n    - [Async learning exercises](#async-learning-exercises)\n    - [Share your feedback](#share-your-feedback)\n\n## Preparations \n\nChoose your [preferred and supported\nIDE](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-in-other-ides-and-editors),\nand follow the documentation to enable code suggestions for [GitLab.com\nSaaS](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-gitlab-saas)\nor [GitLab self-managed\ninstances](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#enable-code-suggestions-on-self-managed-gitlab).\n\n\nProgramming languages can require an install of the language interpreter\ncommand-line tools or compilers that generate binaries from source code to\nbuild and run the application.\n\n\nTip: You can also use [GitLab Remote Development\nworkspaces](/blog/quick-start-guide-for-gitlab-workspaces/) to\ncreate your own cloud development environments, instead of local development\nenvironments. This blog post focuses on using VS Code and the GitLab Web\nIDE. \n\n\n### VS Code\n\nOn macOS, you can [install VS Code](https://code.visualstudio.com/download)\nas a Homebrew cask or manually download and install it. \n\n\n```shell\n\nbrew install --cask visual-studio-code \n\n```\n\n\nNavigate to the `Extensions` menu and search for `gitlab workflow`. Install\nthe [GitLab workflow extension for VS\nCode](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow). \n\n\nTip: VS Code will also detect the programming languages, and offer to\ninstall additional plugins for syntax highlighting and development\nexperience. \n\n\n### Code Suggestions\n\nIt can help to familiarize yourself with suggestions before actually\nverifying the suggestions. GitLab Code Suggestions are provided as you type,\nso you do not need use specific keyboard shortcuts. To accept a code\nsuggestion, press the `tab` key. Also note that writing new code works more\nreliably than refactoring existing code. AI is non-deterministic, which\nmeans that the same suggestion may not be repeated after deleting the code\nsuggestion. While Code Suggestions is in Beta, we are working on improving\nthe accuracy of generated content overall. Please review the [known\nlimitations](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#known-limitations),\nas this could affect your learning experience. \n\n\n## Learning a new programming language: Rust \n\nNow, let's dig into learning Rust, which is one of the [supported languages\nin Code\nSuggestions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html#supported-languages). \n\n\n[Rust by Example](https://doc.rust-lang.org/rust-by-example/) provides a\ngreat tutorial for beginners, together with the official [Rust\nbook](https://doc.rust-lang.org/book/). The [Hands-on Rust\nbook](https://hands-on-rust.com/) shows how to build a 2D game as a more\npractical approach. More examples are shared in [this Rust book\nlist](https://github.com/sger/RustBooks). \n\n\nBefore diving into the source code, make sure to set up your development\nenvironment.\n\n\n### Development environment for Rust\n\n1) Create a new project `learn-rust-ai` in GitLab, and clone the project\ninto your development environment. All code snippets are available in [this\n\"Learn Rust with AI\"\nproject](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai).\n\n\n```shell\n\ngit clone https://gitlab.com/NAMESPACE/learn-rust-ai.git\n\n\ncd learn-rust-ai\n\n\ngit status\n\n```\n\n\n2) Install Rust and the build toolchain. Fortunately, this is\nstraightforward [following the Rust install\ndocumentation](https://www.rust-lang.org/tools/install).\n\n\nTip for using the generic installer: Download the script and run it after\nreview. \n\n\n```\n\n# Download and print the script before running it\n\ncurl -Lvs https://sh.rustup.rs -o rustup-init.sh\n\n\n# Run the Rust installer script\n\nsh rustup-init.sh \n\n```\n\n\nExample on macOS using Homebrew:\n\n\n```\n\nbrew install rust\n\n```\n\n\n1) Optional: Install the [rust-analyzer VS Code\nextension](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer).\n\n\n2) Each exercise will invite you to compile the code with the [`rustc`\ncommand](https://doc.rust-lang.org/rustc/what-is-rustc.html), and later\nusing [`cargo` as build tool and package\nmanager](https://doc.rust-lang.org/cargo/index.html).\n\n\nYou are all set to learn Rust! \n\n\n### Hello, World\n\nWe will start with [Rust by\nExample](https://doc.rust-lang.org/rust-by-example/), and follow the [Hello,\nWorld exercise](https://doc.rust-lang.org/rust-by-example/hello.html).\n\n\nCreate a new file `hello.rs` in the root directory of the project and start\nwith a comment saying `// Hello world`. Next, start writing the `main`\nfunction, and verify the code suggestion.\n\n\n![VS Code hello.rs Rust code suggestion, asking to\naccept](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_hello_world_suggested.png){:\n.shadow}\n\n\nAccept the suggestion by pressing the `tab` key and save the file (keyboard\nshortcut: cmd s). \n\n\n```\n\n// Hello world\n\n\nfn main() {\n    println!(\"Hello, world!\");\n}\n\n```\n\n\n![VS Code hello.rs Rust code suggestion,\naccepted](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_hello_world_accepted.png){:\n.shadow}\n\n\nCommit the change to the Git repository. In VS Code, use the keyboard\nshortcut `ctrl shift G`, add a commit message, and hit `cmd enter` to\nsubmit. \n\n\nUse the command palette (`cmd shift p`) and search for `create terminal` to\nopen a new terminal. \n\n\nBuild and run the code.\n\n\n```shell\n\nrustc hello.rs\n\n\n./hello\n\n```\n\n\n![hello.rs Rust code suggestion, accepted, compiled,\nrun](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_hello_world_cli_build.png){:\n.shadow}\n\n\nTip: Adding [code comments in Rust\n(`//`)](https://doc.rust-lang.org/reference/comments.html) before you start\nwriting a function or algorithm will help Code Suggestions with more context\nto provide better suggestions. In the example above, we did that with `//\nHello world`, and will continue doing so in the next exercises. \n\n\n## Cargo: Bringing structure into Rust\n\n[Cargo](https://doc.rust-lang.org/rust-by-example/cargo.html) is the\nofficial Rust package management tool. It is more than that - you can run\nbuild and test commands because Cargo understands them as well. \n\n\nYou can initialize a new Cargo configuration in the current directory tree\nwith the following command:\n\n\n```shell\n\ncargo init\n\n```\n\n\nThe directory tree invites you to add the source code into the `src/`\ndirectory, while `Cargo.toml` manages the dependencies and used compiler\nversions. The `.gitignore` file is also added including best practices. \n\n\n```shell\n\ntree\n\n.\n\n├── Cargo.toml\n\n├── README.md\n\n├── hello\n\n├── hello.rs\n\n└── src\n    └── main.rs\n```\n\n\nTry building the code and running it using `cargo`.\n\n\n```shell\n\ncargo build\n\n\ncargo run\n\n```\n\n\nCommit all changes and push them to your GitLab project.\n\n\n```shell\n\ngit commit -avm \"Initialize Cargo\"\n\n\ngit push\n\n```\n\n\nAfter exploring Cargo, let's make sure that our code is continuously tested\nwhile learning Rust. The next section explains how to set up [GitLab\nCI/CD](https://about.gitlab.com/topics/ci-cd/) for Rust. \n\n\n## Automation: Configure CI/CD pipeline for Rust\n\nThe [CI/CD pipeline](https://docs.gitlab.com/ee/ci/) should run two jobs in\ntwo stages: Build and test the code. The default container\n[image](https://docs.gitlab.com/ee/ci/yaml/#image), `rust:latest`, works in\nthe first iteration. In order to save resources, the CI/CD configuration\nalso supports [caching](https://docs.gitlab.com/ee/ci/caching/) for\ndownloaded dependencies and build objects. The `CARGO_HOME` variable is set\nto the CI/CD job home directory to ensure everything gets appropriately\ncached.\n\n\n```yaml\n\nstages:\n  - build\n  - test \n\ndefault:\n  image: rust:latest\n  cache:\n    key: ${CI_COMMIT_REF_SLUG}\n    paths:                      \n      - .cargo/bin\n      - .cargo/registry/index\n      - .cargo/registry/cache\n      - target/debug/deps\n      - target/debug/build\n    policy: pull-push\n\n# Cargo data needs to be in the project directory to be cached. \n\nvariables:\n  CARGO_HOME: ${CI_PROJECT_DIR}/.cargo      \n```\n\n\nThe CI/CD jobs inherit the\n[`default`](https://docs.gitlab.com/ee/ci/yaml/#default) values, and specify\nthe cargo commands in the [`script`\nsection](https://docs.gitlab.com/ee/ci/yaml/#script).\n\n\n```yaml\n\nbuild-latest:\n  stage: build\n  script:\n    - cargo build --verbose\n\ntest-latest:\n  stage: build\n  script:\n    - cargo test --verbose\n```\n\n\nYou can see an example in [this\nMR](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai/-/merge_requests/1/diffs).\n\n\n## Continue learning Rust \n\nMake sure to add new source code into the `src/` directory. \n\n\n### Define variables and print them\n\nPractice adding a few more\n[print](https://doc.rust-lang.org/rust-by-example/hello/print.html)\nstatements into `src/main.rs`, and then build and run the code again.\n\n\n1) Define a variable called `name` and assign your name as string value.\n\n\n2) Print the name, including a string prefix saying `Hello, `. \n\n\n![VS Code main.rs Rust code suggestion, first step in\nprint](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_print_variable_first.png){:\n.shadow}\n\n\n![VS Code main.rs Rust code suggestion, second step in\nprint](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_print_variable_second.png){:\n.shadow}\n\n\n1) Open a new terminal in VS Code using the command palette (keyboard\nshortcut `cmd + shift + p`) and search for `terminal`.\n\n\n2) Build and run the code with the `cargo build` and `cargo run` commands. \n\n\n![VS Code terminal with cargo build and run\noutput](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_print_variable_cargo_build_run_terminal.png){:\n.shadow}\n\n\nAn example solution can be found\n[here](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai/-/blob/main/solutions/variable_print.rs). \n\n\n### Explore variable types \n\nDefine different variable value types\n([primitives](https://doc.rust-lang.org/rust-by-example/primitives.html))\nand embed them into the `print` statements. Maybe they feel familiar with\nother programming languages?\n\n\nTip: Use code comments to see which code suggestions can be useful to learn.\nStart with typing `// Integer addition` and see what code suggestions you\ncan add.\n\n\n![VS Code main.rs Rust code suggestion, primitive types with literals and\nexpressions](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_primitive_types_literals_operators.png)\n\n\nExperiment with GitLab Duo Code Suggestions. The shown examples are\nnon-deterministic, but you may be able to add additions, subscriptions,\nmultiplications, etc., and the corresponding `println` statements just by\naccepting code suggestions and continuing the flow with `enter` or\ncompleting the code statements. This workflow can create a chain of code\nsuggestions that can help you learn the Rust language. \n\n\n![Literals and expressions, first\nsuggestion](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_literals_expressions_01.png){:\n.shadow}\n\n![Literals and expressions, second\nsuggestion](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_literals_expressions_02.png){:\n.shadow}\n\n![Literals and expressions, third\nsuggestion](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_literals_expressions_03.png){:\n.shadow}\n\n\nAn example solution can be found\n[here](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai/-/blob/main/solutions/literals_expressions.rs). \n\n\nThe code suggestions are not perfect. Sometimes there are errors that\nrequire you to fix the problems. When writing this blog post, I had to fix\ntwo missing semicolons at the end of the code lines. The great thing about\nthe Rust compiler is that the error messages tell you exactly where the\nproblem happens with suggestions to fix them. Code Suggestions and the\nRust-provided build chain make writing Rust code more efficient. \n\n\n```rust\n\nprintln!(\"Hello, {}!\", name)\n\n\n// Integer subtraction\n\nlet y = 9 - 4\n\n```\n\n\n![Terminal build, errors, Rust compiler\nhelp](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_terminal_errors_rust_help.png){:\n.shadow}\n\n\nYou can try to provoke the same error by removing a semicolon at the end of\na statement and then running `cargo build` in the terminal again. The Rust\ncompiler will also warn you about unused variables to help with better code\nquality. The screenshot shows warnings for variable definitions, and also a\nCLI command to fix them. \n\n\n![Terminal build, warnings, Rust compiler\nhelp](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_terminal_warnings_rust_help.png){:\n.shadow}\n\n\n### Flow control: Conditions and loops \n\nNext, let's focus on [flow\ncontrol](https://doc.rust-lang.org/rust-by-example/flow_control.html) with\nconditions, loops, etc., and how to implement them.\n\n\n1) Start typing `// Flow control` and see which suggestions are provided.\n\n\n2) Experiment with the code, and continue defining a boolean variable `v`\nwhich is set to true. \n\n\n```rust\n  // Flow control\n  let v = true;\n\n```\n\n\n![Conditions, boolean\nvariable](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_flow_control_conditions_01.png){:\n.shadow}\n\n![Conditions, boolean variable, if\ncondition](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_flow_control_conditions_02.png){:\n.shadow}\n\n\n1) Start typing `// Loops` and experiment with the code suggestions. \n\n\nLet's assume the loop looks the like following snippet. It does not have a\nloop counter which gets printed on every loop execution.\n\n\n```rust\n\n// Loops\n\nlet mut count = 0;\n\n\nloop {\n    count += 1;\n\n    if count == 10 {\n        break;\n    }\n}\n\n```\n\n\n2) Start typing `println!` and see which code suggestions are provided, for\nexample `println!(\"Count: {}\", count);`. \n\n\n![Loops, loop counter print\nsuggestion](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_flow_control_loops_print_counter.png)\n\n\n3) Apply the suggestions, and execute `cargo build && cargo run` on the\nterminal again. \n\n\nLet's learn more: Rust supports different loop types, for example [while\nloops](https://doc.rust-lang.org/rust-by-example/flow_control/while.html),\n[for\nloops](https://doc.rust-lang.org/rust-by-example/flow_control/for.html),\netc. \n\n\n1) Type `// While loop` and verify the code suggestions. Repeat the same for\n`// For loop`.\n\n\n```rust\n\n// While loops\n\nlet mut count = 0;\n\n\nwhile count \u003C 10 {\n    count += 1;\n    println!(\"Count: {}\", count);\n}\n\n\n// For loops\n\nlet a = [10, 2, 3, 4, 5];\n\n\nfor element in a {\n    println!(\"Element: {}\", element);\n}\n\n```\n\n\nThere is more to learn with loops and conditions: Iterate over arrays,\nlists, maps, slices. Practice with writing comments for `// Maps and sets`\nand `// Vectors and strings`. \n\n\n![Vectors,\nstrings](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_flow_control_vectors_strings.png){:\n.shadow}\n\n\n```rust\n  // Maps and sets\n  let mut scores = HashMap::new();\n\n  scores.insert(String::from(\"Blue\"), 10);\n  scores.insert(String::from(\"Yellow\"), 50);\n\n  for (key, value ) in &scores {\n      println!(\"{}: {}\", key, value);\n  }\n\n  // Vectors and strings\n  let mut v = Vec::new();\n\n  v.push(1);\n  v.push(2);\n\n  for element in &v {\n      println!(\"Element: {}\", element);\n  }  \n```\n\n\nThis snippet will fail because the `HashMap` type needs to be imported from\n`std::collections::HashMap`. Add the following line on top before the main\nfunction definition: \n\n\n```rust\n\nuse std::collections::HashMap;\n\n``` \n\n\n2) Build and run the code with `cargo build && cargo run`. \n\n\nAn example solution is provided\n[here](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai/-/blob/main/solutions/flow_control.rs).\n\n\n**Async exercise**: \n\n\n1) Modify the input values for the different data types, and build and run\nthe code again.\n\n\n2) Add a condition into the loops that print the items only when a specific\ncondition is met (for example, the number is odd or even). \n\n\n### Functions \n\n\n[Functions](https://doc.rust-lang.org/rust-by-example/fn.html) help increase\ncode readability and testability with unit tests. Practice creating\nfunctions with the following steps: \n\n\n1) Two functions `isEven` and `isOdd` to evaluate whether a number is even\nor odd.\n\n\n```rust\n\nfn isEven(x: i32) -> bool {\n    x % 2 == 0\n}\n\n\nfn isOdd(x: i32) -> bool {\n    x % 2 != 0\n}\n\n```\n\n\n2) `isPrime` function to check whether a given integer value is a prime\nnumber.\n\n\n```rust\n\nfn isPrime(x: i32) -> bool {\n    let mut i = 2;\n\n    while i * i \u003C= x {\n        if x % i == 0 {\n            return false;\n        } else {\n            i += 1;\n        }\n    } \n\n    return true\n}\n\n```\n\n\n3) Create an array of integer values, loop over it, and call the functions.\nLet GitLab Code Suggestions guide you with the implementation by starting to\ntype the if conditions followed by the function name. \n\n\n```rust\n  // Functions\n  let mut integers = vec![1, 2, 3, 4, 5];\n\n  for i in integers.iter() {\n\n      if (isEven(i)) {\n          println!(\"{} is even\", i);\n      }\n\n      if (isOdd(i)) { \n          println!(\"{} is odd\", i);\n      }\n\n      if (isPrime(i)) { \n          println!(\"{} is prime\", i);\n      }\n\n      println!(\"{}\", i);\n  }\n```\n\n\nNote that passing a reference value to a function may result in an error\nfrom the Rust compiler. Follow the suggestions and build the code again. \n\n\n```shell\n\n$ cargo build && cargo run \n\n\nerror[E0308]: mismatched types\n   --> src/main.rs:112:21\n    |\n112 |         if (isPrime(i)) { \n    |             ------- ^ expected `i32`, found `&{integer}`\n    |             |\n    |             arguments to this function are incorrect\n    |    \nnote: function defined here\n   --> src/main.rs:136:4\n    |\n136 | fn isPrime(x: i32) -> bool {\n    |    ^^^^^^^ ------\nhelp: consider dereferencing the borrow\n    |\n112 |         if (isPrime(*i)) { \n    |                     +\n```\n\n\nAn example solution is provided\n[here](https://gitlab.com/gitlab-de/use-cases/ai/learn-with-ai/learn-rust-ai/-/blob/main/solutions/functions.rs).\n\n\n**Async exercise**: Create a function `containsString` and test it with an\narray of string values, and a string to search for, in a loop. The\nscreenshot shows a potential implementation. \n\n\n![containsString function, and vector with string elements to test,\nsuggesting its usage in the main\nfunction](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_function_implemented_then_suggested_in_main.png){:\n.shadow}\n\n\n### Testing \n\nWhile learning programming, adopt\n[testing](https://doc.rust-lang.org/rust-by-example/testing.html) into your\nprocess. This can be unit tests for functions, documentation testing, and\nintegration testing. Practice with testing the previously created functions\n`isEven`, `isOdd`, and `isPrime`. Starty by typing `mod tests {` followed by\na new line with `use super::*` to implement the example from the [Rust\ndocumentation for unit\ntests](https://doc.rust-lang.org/rust-by-example/testing/unit_testing.html).\n\n\n```rust\n\nmod tests {\n    use super::*;\n\n    #[test]\n    fn test_is_even() {\n        assert!(isEven(2)); \n        assert!(!isEven(3));\n    }\n\n    #[test] \n    fn test_is_odd() {\n        assert!(!isOdd(2));\n        assert!(isOdd(3));\n    }\n\n    #[test]\n    fn test_is_prime() { \n        assert!(isPrime(2));\n        assert!(!isPrime(3));\n    }\n}\n\n```\n\n\nRun `cargo test` to run the unit tests. Modify the test values to experiment\nwith the results. \n\n\n```shell\n\ncargo test\n\n```\n\n\n![Function unit tests, cargo test output in the VS Code\nterminal](https://about.gitlab.com/images/blogimages/learn-rust-with-ai-code-suggestions-getting-started/learn_rust_ai_gitlab_code_suggestions_function_unit_tests_terminal_run.png)\n\n\nCreate unit tests that fail, and commit and push the changes to GitLab. The\nCI/CD pipelines will fail in this simulated breakage. The example above\nneeds a fix for the `test_is_prime` test. Commit and push the change to\nverify that the pipeline passes again. \n\n\n```diff\n\n-        assert!(!isPrime(3));\n\n+        assert!(!isPrime(4));\n\n```\n\n\n## What is next \n\nIn an upcoming blog, we will look into advanced learning examples with\nasynchronous operations, services and external API communication in future\nblog posts. Until then, here are a few recommendations for practicing async.\n\n\n### Async learning exercises\n\n- [`std misc`](https://doc.rust-lang.org/rust-by-example/std_misc.html)\nprovides asynchronous operations with threads, channels and file I/O\n\n- Book: [Hands-on Rust: Effective Learning through 2D Game Development and\nPlay](https://pragprog.com/titles/hwrust/hands-on-rust/)\n\n- Tutorial: [Are we game yet?](https://arewegameyet.rs/resources/tutorials/)\n\n- Use case: [Web server with\nrocket.rs](https://rocket.rs/v0.5-rc/guide/quickstart/#running-examples)\n\n\nHere are a few more exercises and ideas for additional learning:\n\n1) The Rust compiler might have created warnings that need to be addressed.\nFollow the instructions from the `cargo build` commands and check the Git\ndiff. \n\n\n```\n\ncargo fix --bin \"learn-rust-ai\"\n\n\ngit diff \n\n```\n\n\n2) [Error handling](https://doc.rust-lang.org/rust-by-example/error.html) is\nrequired when failure is detected, and the caller should know. Some errors\ncan be recovered from within the application, others require program\ntermination. \n\n\n3) The [`std` library](https://doc.rust-lang.org/rust-by-example/std.html)\nextends primitive types and makes programming more efficient. \n\n\n### Share your feedback\n\nWhich programming language are you learning or considering learning? Start a\nnew topic on our [community](/community/) forum or Discord and share your\nexperience.  \n\n\nIf you are using Code Suggestions Beta with [GitLab Duo](/gitlab-duo/)\nalready, please share your thoughts and feedback [in this\nissue](https://gitlab.com/gitlab-org/gitlab/-/issues/405152).\n",[851,1121,9,956,720],{"slug":3321,"featured":6,"template":700},"learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started","content:en-us:blog:learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started.yml","Learning Rust With A Little Help From Ai Code Suggestions Getting Started","en-us/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started.yml","en-us/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started",{"_path":3327,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3328,"content":3334,"config":3339,"_id":3341,"_type":13,"title":3342,"_source":15,"_file":3343,"_stem":3344,"_extension":18},"/en-us/blog/mastering-gitlab-admin-tasks-with-gitlab-duo-chat",{"title":3329,"description":3330,"ogTitle":3329,"ogDescription":3330,"noIndex":6,"ogImage":3331,"ogUrl":3332,"ogSiteName":685,"ogType":686,"canonicalUrls":3332,"schema":3333},"Mastering GitLab admin tasks with GitLab Duo Chat","Learn how to use Chat to streamline administrative tasks on self-managed instances, improving efficiency and problem-solving capabilities.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666405/Blog/Hero%20Images/GitLab_Duo_Blog_Hero_1800x945_r2_B__1_.png","https://about.gitlab.com/blog/mastering-gitlab-admin-tasks-with-gitlab-duo-chat","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Mastering GitLab admin tasks with GitLab Duo Chat\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"David O'Regan\"}],\n        \"datePublished\": \"2024-08-09\",\n      }",{"title":3329,"description":3330,"authors":3335,"heroImage":3331,"date":3336,"body":3337,"category":718,"tags":3338},[2383],"2024-08-09","As a GitLab administrator managing a self-hosted instance, you often face\ncomplex challenges that require innovative solutions. Enter [GitLab Duo\nChat](https://about.gitlab.com/gitlab-duo/) – your AI-powered assistant that\ncan significantly streamline your administrative tasks. In this article,\nwe'll explore how you can leverage GitLab Duo Chat to solve intricate\nproblems efficiently, using a real-world example of updating group\nmemberships across multiple groups.\n\n\n## The power of GitLab Duo Chat for admins\n\n\nGitLab Duo Chat is more than just conversational AI; it's a powerful tool\nthat can assist with complex administrative tasks. By providing\ncontext-aware suggestions and code snippets, Chat can help you navigate\nthrough GitLab's extensive feature set and underlying architecture.\n\n\n### Case study: Updating group memberships\n\n\nLet's dive into a scenario where an admin needs to add an administrator user\nto multiple [groups](https://docs.gitlab.com/ee/user/group/) – in this case,\n50,000 groups. This task, while conceptually simple, can be daunting due to\nits scale.\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/dBd957MK_DE?si=JYTzdRjVQHyB6rpl\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Step-by-step problem-solving with GitLab Duo Chat\n\n\n* **Initial approach:** Our first instinct might be to use GitLab's API, but\nwith 50,000 groups, this could put significant strain on the system.\n\n* **Pivoting to the Rails console:** Instead, we can use the Rails console\nfor a more efficient solution. Here's where Chat comes in handy.\n\n* **Crafting the solution:** Let's walk through the process of creating a\nRuby script to accomplish our task:\n\n\n``` python\n\n\ndef add_admin_to_groups(parent_group, admin_user)\n     unless parent_group.members.exists?(user_id: admin_user.id)\n       parent_group.add_owner(admin_user)\n       puts \"Added #{admin_user.name} to #{parent_group.name}\"\n     end\n\n    parent_group.descendants.each do |subgroup|\n      add_admin_to_groups(subgroup, admin_user)\n    end\n  end\n\n# Usage\n\n\nadmin_user = User.find_by(username: 'mr_admin')\n\nparent_group = Group.find_by(path: 'top_group')\n\nadd_admin_to_groups(parent_group, admin_user)\n\n```\n\n\n* **Iterative refinement:** Through conversation with Chat, we refined this\nscript, addressing issues like method availability and parameter\nrequirements.\n\n\n### Key takeaways\n\n\n#### Non-deterministic responses\n\n\nGitLab Duo Chat may provide different answers to the same question,\nencouraging exploration of multiple solutions. As an example:\n\n\nAn admin asks, ***How can I improve CI/CD pipeline performance?***\n\n\nChat's first response could be:\n\n***Consider using GitLab CI/CD caching to speed up subsequent pipeline runs.\nHere's an example .gitlab-ci.yml snippet:***\n\n\n```\n\ncache:\n  key: ${CI_COMMIT_REF_SLUG}\n  paths:\n    - node_modules/    \n ```\n\n***This caches the `node_modules` directory, reducing build times for\nNode.js projects.***\n\n\nWhen asked again, Chat's second response could be: ***To improve CI/CD\nperformance, try parallelizing your jobs. Here's an example:***\n\n\n```\n\ntest:\n  parallel: 3\n```\n\n\n***This runs your test job across three parallel runners, potentially\nreducing overall pipeline duration.***\n\n\n### Error interpretation and debugging\n\n\nChat can help interpret error messages and suggest fixes. As an example, an\nadmin encounters an error: \"fatal: reference is not a tree: 8a7d460\". They\ncould ask Chat: ***I'm seeing this Git error. What does it mean and how can\nI fix it?***\n\n\nChat could respond with:\n\n\n***This error typically occurs when Git can't find a specific commit. It\ncould be due to a corrupted repository or incomplete fetch. Try these\nsteps:***\n\n* ***Run `git fsck` to check for corruption.***\n\n* ***If that doesn't work, try git fetch --all --prune to update all\nremotes.***\n\n* ***If the issue persists, you might need to re-clone the repository.***\n\n\n***Always back up your repository before attempting these fixes.***\n\n\n### Code optimization and refactoring\n\n\nChat can assist in optimizing and refactoring code for better performance.\n\n\nAn example: An admin wants to optimize a Gitaly configuration for a large\ninstance. They could ask Chat: ***How can I optimize Gitaly for an instance\nwith 10,000+ repositories?***\n\n\nChat could respond with: ***For a large instance, consider this optimized\nGitaly configuration:***\n\n\n```\n\n[concurrency]\n\nrpc_concurrency = 100\n\n\n[rate_limiting]\n\nrpc_rate_limit = 100\n\n\n[cgroups]\n\nmountpoint = '/sys/fs/cgroup'\n\nhierarchy_root = 'gitaly'\n\nmemory_bytes = 30000000000  # Adjust based on available memory\n\ncpu_shares = 1024\n\n```\n\n***This configuration increases concurrency, implements rate limiting to\nprevent overload, and uses cgroups (control groups) to manage resource\nallocation. Adjust values based on your specific hardware and needs.***\n\n\n## Best practices for using GitLab Duo Chat in admin tasks\n\n\n### Formulate queries clearly and provide context\n\n\nWhen using Chat, the quality of the response largely depends on the quality\nof your query. Clear, detailed questions that provide relevant context will\nyield more accurate and helpful answers.\n\n\nPoor query: ***Why is GitLab slow?***\n\n\nThis query lacks specifics and context, making it difficult for Chat to\nprovide a targeted response. \n\n\nA better query would be: ***Our GitLab instance with 5,000 users and 3,000\nprojects is experiencing slow response times, especially during peak hours\n(9-11 AM EST). CPU usage on the application servers spikes to 90%. How can\nwe diagnose and address this?***\n\n\nThis improved query provides crucial details:\n\n\n* scale of the instance (5,000 users, 3,000 projects)\n\n* nature of the problem (slow response times)\n\n* timing of the issue (peak hours, 9-11 AM EST)\n\n* observed symptoms (90% CPU spike)\n\n\nWith this information, Chat can provide more targeted advice.\n\n\nAn even better query would be: ***We're running GitLab 15.8.3 on a 3-node\ncluster (8 vCPUs, 32GB RAM each) with a separate PostgreSQL 13 database and\nRedis 6.2 instance. Our instance hosts 5,000 users and 3,000 projects. We're\nexperiencing slow response times (average 5s, up from our usual 1s) during\npeak hours (9-11 AM EST), primarily affecting merge request creation and\npipeline initiation. CPU usage on the application servers spikes to 90%,\nwhile database CPU remains under 60%. Gitaly CPU usage is around 70%. We've\nalready increased Puma workers to 8 per node. What additional diagnostics\nshould we run and what potential solutions should we consider?***\n\n\nThis query provides an extensive context, including:\n\n* GitLab version and infrastructure details\n\nspecific performance metrics (response time increase)\n\n* affected operations (merge requests, pipelines)\n\n* resource usage across different components\n\n* steps already taken to address the issue\n\n\nBy providing this level of detail, you enable Chat to:\n\n* understand the full scope of your environment\n\n* identify potential bottlenecks more accurately\n\n* suggest relevant diagnostic steps\n\n* propose solutions tailored to your specific setup\n\n\nAvoid recommending steps you've already taken.\n\n\nRemember, while GitLab Duo Chat is powerful, it's not omniscient. The more\nrelevant information you provide, the better it can assist you. By following\nthese guidelines, you'll get the most out of your interactions with Chat,\nleading to more effective problem-solving and administration of your GitLab\ninstance.\n\n\n### Use GitLab Duo Chat's suggestions as a starting point and refine\nincrementally\n\n\nChat is an excellent tool for getting started with complex tasks, but it's\nmost effective when used as part of an iterative process. Begin with a broad\nquestion, then use Chat's responses to guide your follow-up questions,\ngradually refining your understanding and solution.\n\n\n#### Initial query\n\n\nAdmin: ***How can I set up Geo replication for disaster recovery?***\n\n\nChat might respond with a basic setup guide, covering:\n\n- prerequisites for Geo setup\n\n- steps to configure the primary node\n\n- process for adding a secondary node\n\n- initial replication process\n\n\nThis provides a foundation, but complex setups like Geo often require more\nnuanced understanding. Here's how you might refine your queries:\n\n\n**- Follow-up Query 1**\n\n\nAdmin: ***How do I handle custom data in Geo replication?***\n\nThis question addresses a specific concern not covered in the initial\nsetup. \n\n\n**- Follow-up Query 2**\n\n\nAdmin: ***What's the best way to test failover without disrupting\nproduction?***\n\n\nThis query focuses on a critical operational concern. \n\n\n**- Follow-up Query 3**\n\n\nAdmin: ***Can you help me create a runbook for Geo failover?***\n\n\nThis final query aims to consolidate the gathered information into a\npractical guide. The benefits of this incremental approach:\n\n\n1. By breaking down the complex topic of Geo replication into smaller,\nfocused queries, you gain a more thorough understanding of the subject.\n\n2. Each follow-up question allows you to address specific concerns relevant\nto your environment, resulting in a more customized solution.\n\n3. The progression from setup to testing to creating a runbook ensures that\nyou're not just understanding the theory, but also preparing for real-world\nimplementation.\n\n4. The step-by-step process of refining your queries helps in better\nretention of the information, as you're actively engaging with the content.\n\n5. Follow-up questions often reveal aspects of the task you might not have\ninitially considered, leading to a more robust final solution.\n\n\n#### Best practices for incremental refinement\n\n\n- Start with broad questions to establish a foundation.\n\n- Use Chat's responses to identify areas that need more clarification or\ndetail.\n\n- Ask about specific scenarios or edge cases relevant to your environment.\n\n- Seek practical advice on implementation, testing, and maintenance.\n\n- Use your growing understanding to formulate increasingly specific and\ncomplex queries.\n\n- Periodically summarize your understanding and ask Chat to validate or\ncorrect it.\n\n\nRemember, while GitLab Duo Chat is a powerful tool, it's most effective when\ncombined with your own critical thinking and domain knowledge. Use it to\nguide your learning and decision-making process, but always validate\nimportant information and test solutions in a safe environment before\napplying them to production systems.\n\n\n### Potential applications for self-managed GitLab instances\n\n\n#### User management at scale\n\n\nAdmin: ***We need to onboard 1,000 new users from an acquired company. The\ndata is in a CSV file. How can we automate this process? I am an Admin, can\nI use a Python script for this?***\n\n\nChat: ***You can use GitLab's API for bulk user creation. Here's a Python\nscript to get you started:***\n\n\n```python\n\nimport requests\n\nimport csv\n\n\ndef create_user(username, email, name, password):\n    url = 'https://gitlab.example.com/api/v4/users'\n    headers = {'PRIVATE-TOKEN': 'your_access_token'}\n    data = {\n        'username': username,\n        'email': email,\n        'name': name,\n        'password': password,\n        'skip_confirmation': True\n    }\n    response = requests.post(url, headers=headers, data=data)\n    return response.status_code == 201\n\nwith open('new_users.csv', 'r') as f:\n    reader = csv.reader(f)\n    for row in reader:\n        username, email, name, password = row\n        if create_user(username, email, name, password):\n            print(f\"Created user: {username}\")\n        else:\n            print(f\"Failed to create user: {username}\")\n```\n\n***This script reads user data from a CSV file and creates users via the\nGitLab API. Remember to handle errors and rate limiting for large\nbatches.***\n\n\n#### Project and group administration\n\n\nAdmin: ***We need to implement a standardized project structure across our\norganization. How can we automate this? Assume I am an admin. Write me a\nscript to do this via the API.***\n\n\nChat: ***You can use GitLab's API to create a project template and then\napply it to new projects. Here's a bash script to create a new project with\na standard structure:***\n\n\n```\n\n#!/bin/bash\n\n\nTOKEN=\"your_access_token\"\n\nGROUP_ID=\"your_group_id\"\n\n\ncreate_project() {\n  curl --header \"PRIVATE-TOKEN: $TOKEN\" \\\n  --data \"name=$1&namespace_id=$GROUP_ID\" \\\n  \"https://gitlab.example.com/api/v4/projects\"\n}\n\n\nadd_file() {\n  curl --request POST --header \"PRIVATE-TOKEN: $TOKEN\" \\\n  --form \"branch=main\" \\\n  --form \"content=$2\" \\\n  --form \"commit_message=Add $1\" \\\n  \"https://gitlab.example.com/api/v4/projects/$3/repository/files/$1\"\n}\n\n\nPROJECT_ID=$(create_project \"New-Standardized-Project\" | jq '.id')\n\n\nadd_file \"README.md\" \"# Welcome to our standardized project\" $PROJECT_ID\n\nadd_file \".gitlab-ci.yml\" \"stages:\\n  - build\\n  - test\\n  - deploy\"\n$PROJECT_ID\n\nadd_file \"CONTRIBUTING.md\" \"## How to contribute to this project\"\n$PROJECT_ID\n\n\necho \"Project created with ID: $PROJECT_ID\"\n\n```\n\n\n***This script creates a new project with a README, a basic CI/CD\nconfiguration, and a CONTRIBUTING guide. You can expand this to include more\nfiles or even clone a template repository.***\n\n\n#### Performance optimization\n\n\nAdmin: ***Our merge request creation process is slow. How can we diagnose\nand improve this? Assume I am an admin.***\n\n\nChat: ***As an instance admin, there are a few potential areas you can\ninvestigate to diagnose and improve the merge request creation process:\n\nDatabase Performance: The provided quotes show some example SQL queries\nrelated to merge request metrics. Slow database queries can significantly\nimpact merge request creation performance. You should review the database\nperformance, indexes, and query plans for any queries involved in the merge\nrequest creation process.***\n\n\n- ***Background jobs: Merge request creation may involve background jobs\n(e.g., for data processing, notifications, etc.). If these jobs are backed\nup or running slowly, it can impact the overall merge request creation time.\nCheck the background job queues and workers.***\n\n- ***Code optimization: Analyze the code responsible for merge request\ncreation to identify any potential bottlenecks or inefficiencies. This could\ninclude things like unnecessary database queries, inefficient algorithms, or\nlack of caching.***\n\n- ***Resource utilization: Monitor the resource utilization (CPU, memory,\ndisk I/O) of the GitLab application servers during merge request creation.\nHigh resource utilization can lead to performance degradation.***\n\n- ***Network performance: If your GitLab instance is distributed across\nmultiple nodes or servers, network performance between these components can\nimpact merge request creation times. Check for any network bottlenecks or\nlatency issues.***\n\n- ***Caching: Ensure that appropriate caching mechanisms are in place and\nfunctioning correctly. Inefficient or incorrect caching can lead to\nunnecessary overhead during merge request creation.***\n\n- ***Load testing: Perform load testing to simulate real-world scenarios and\nidentify any performance bottlenecks or scalability issues that may arise\nunder high load.***\n\n- ***Logging and monitoring: Enable detailed logging and monitoring for the\nmerge request creation process. This can help identify specific areas of\nslowness or bottlenecks.***\n\n\n***To effectively diagnose and improve the merge request creation process,\nyou'll likely need to combine multiple techniques, such as profiling, code\nanalysis, database optimization, and load testing. Additionally, consulting\nthe GitLab documentation and community resources can provide valuable\ninsights and best practices specific to your GitLab version and\nconfiguration.***\n\n\n#### Troubleshooting complex issues\n\n\nAdmin: ***We're experiencing intermittent 502 errors on our GitLab instance.\nHow can we troubleshoot this? I am an instance admin, how can I check the\nlogs to find answers?***\n\n\nChat: ***To troubleshoot intermittent 502 errors on your GitLab instance as\nan admin, you can follow these steps:***\n\n\n***1. Check the main GitLab logs while reproducing the issue by running:***\n\n\n```\n\nsudo gitlab-ctl tail | tee /tmp/\u003Ccase-ID-and-keywords>.log\n\n```\n\n\n***2. Press Ctrl+C to stop tailing the logs.***\n\n\n***3. Run GitLabSOS to gather broader diagnostics about your instance:***\n\n\n```\n\nsudo /opt/gitlab/embedded/bin/gitlab-sos\n\n```\n\n\n***4. Use the fast-stats tool to analyze performance statistics from the\nlogs:***\n\n\n```\n\nsudo /opt/gitlab/embedded/bin/fast-stats\n/var/log/gitlab/gitlab-rails/production.log\n\n```\n\n\n***5.Look for relevant log entries using the correlation ID that is present\nin most requests.***\n\n\n***6. Check the Pages logs (gitlab_pages_error.log) for any specific errors\nrelated to Pages communication issues with the GitLab API or loopback\ntimeouts if using an AWS Network Load Balancer.***\n\n\n***7. Ensure your gitlab-secrets.json file is synced across all GitLab Pages\ninstances if using `domain_config_source=auto`.***\n\n\n***8. If using systemd, configure GitLab Pages to listen on an explicit IP\nand port to avoid /tmp being cleaned up.***\n\n\n## Get started with GitLab Duo Chat\n\n\nGitLab Duo Chat is a powerful ally for administrators of self-managed GitLab\ninstances. By leveraging its capabilities, you can tackle complex tasks more\nefficiently, learn new techniques, and ultimately become a more effective\nGitLab administrator.\n\n\nWe encourage you to experiment with Chat in your administrative workflows.\nRemember to use it responsibly and always verify the solutions it provides.\n\n\n> [Try GitLab Duo for free](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/).\n\n\n### Resources\n\n- [GitLab Duo documentation](https://docs.gitlab.com/ee/user/gitlab_duo/)\n\n- [GitLab Rails Console Cheat\nSheet](https://docs.gitlab.com/ee/administration/operations/rails_console.html)\n\n- [GitLab API documentation](https://docs.gitlab.com/ee/api/)\n\n- [10 best practices for using AI-powered GitLab Duo\nChat](https://about.gitlab.com/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat/)\n\n- [GitLab Duo Chat 101: Get more done on GitLab with our AI\nassistant](https://about.gitlab.com/blog/gitlab-duo-chat-101-get-more-done-on-gitlab-with-our-ai-assistant/)\n",[720,9,495,696,694],{"slug":3340,"featured":90,"template":700},"mastering-gitlab-admin-tasks-with-gitlab-duo-chat","content:en-us:blog:mastering-gitlab-admin-tasks-with-gitlab-duo-chat.yml","Mastering Gitlab Admin Tasks With Gitlab Duo Chat","en-us/blog/mastering-gitlab-admin-tasks-with-gitlab-duo-chat.yml","en-us/blog/mastering-gitlab-admin-tasks-with-gitlab-duo-chat",{"_path":3346,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3347,"content":3351,"config":3359,"_id":3361,"_type":13,"title":3362,"_source":15,"_file":3363,"_stem":3364,"_extension":18},"/en-us/blog/measuring-ai-roi-at-scale-a-practical-guide-to-gitlab-duo-analytics",{"config":3348,"title":3349,"description":3350},{"noIndex":6},"Measuring AI ROI at scale: A practical guide to GitLab Duo Analytics","Learn how to transform raw usage data into actionable business insights and ROI calculations with this in-depth tutorial.",{"title":3349,"description":3350,"authors":3352,"heroImage":3354,"body":3355,"category":718,"tags":3356,"date":3357,"updatedDate":3358},[3353],"Paul Meresanu","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662840/Blog/Hero%20Images/ai-experiment-stars.png","AI investment starts with measurement. Building a successful AI-powered\ndevelopment platform begins with understanding actual usage, adoption\npatterns, and quantifiable business value — especially ROI from [GitLab Duo\nEnterprise](https://about.gitlab.com/gitlab-duo/).\n\n\nTo help our customers maximize their AI investments, we developed the GitLab Duo Analytics solution as part of our Duo Accelerator program — a comprehensive, customer-driven solution that transforms raw usage data into actionable business insights and ROI calculations. This is not a GitLab product, but rather a specialized enablement tool we created to address immediate analytics needs while organizations transition toward comprehensive AI productivity measurement.\n\n\nThis foundation enables broader AI transformation. For example, organizations can use these insights to optimize license allocation, identify high-value use cases, and build compelling business cases for expanding AI adoption across development teams.\n\n\nA leading financial services organization partnered with a GitLab customer success architect through the Duo Accelerator program to gain visibility into their GitLab Duo Enterprise investment. Together, we implemented a hybrid analytics solution that combines monthly data collection with real-time API integration, creating a scalable foundation for measuring AI productivity gains and optimizing license utilization at enterprise scale.\n\n\n## The challenge: Measuring AI ROI in enterprise development\n\n\nBefore implementing any analytics solution, it's essential to understand your AI measurement landscape.\n\n\nConsider:\n\n\n- **What GitLab Duo features need measurement?** (code suggestions, chat assistance, security scanning)?\n\n- **Who are your AI users?** (developers, security teams, DevOps engineers)?\n\n- **What business metrics matter?** (time savings, productivity gains, cost optimization)?\n\n- **How does your current data collection work** (manual exports, API integration, existing tooling)?\n\n\nUse this stage to define your:\n\n\n- ROI measurement framework\n\n- Key performance indicators (KPIs)\n\n- Data collection strategy\n\n- Stakeholder reporting requirements\n\n\n### Sample ROI measurement framework\n\n\n![Sample ROI measurement framework](https://gitlab.com/-/project/54775568/uploads/06da2f5c3a75197cd272aedb3d67a347/image.png)\n\n\n## Step-by-step implementation guide\n\nImportant: The solution below describes an open source approach that you can deploy in your own environment. It is **NOT** a commercial product from GitLab that you need to purchase. You should be able download, customize, and run this solution free of charge.\n\n### Prerequisites\n\n**Before starting, ensure you have:**\n\n- GitLab instance with Duo enabled\n\n- GitLab API token with read permissions\n\n- Access to configure GitLab CI/CD variables\n\n- Basic understanding of GitLab CI/CD pipelines\n\n\n### 1. Initial setup and configuration\n\nLet's set up the project environment by first cloning the repository.\n\n```bash\ngit clone https://gitlab.com/gitlab-org/professional-services-automation/tools/utilities/gitlab-graphql-api.git\ncd gitlab-graphql-api\n```\nThen push it to your own GitLab instance or namespace, and navigate to your project in GitLab to configure the CI/CD variables.\n\n### 2. Configure pipeline control variables \n\nControl which analytics pipelines run by setting Project CI/CD Variables. \nGo to **Project Settings → CI/CD → Variables** and add:\n\n| Variable | Default | Description |\n|----------|---------|-------------|\n| `ENABLE_DUO_METRICS` | \"true\" | Enable/disable Duo AI metrics pipeline |\n| `ENABLE_PROJECT_METRICS` | \"false\" | Enable/disable traditional project metrics pipeline |\n\n#### Example configurations:\n\n- **Duo Only**: `ENABLE_DUO_METRICS=\"true\"`, `ENABLE_PROJECT_METRICS=\"false\"`\n- **Both Dashboards**: `ENABLE_DUO_METRICS=\"true\"`, `ENABLE_PROJECT_METRICS=\"true\"`\n- **Disable All**: `ENABLE_DUO_METRICS=\"false\"`, `ENABLE_PROJECT_METRICS=\"false\"`\n\n### 3. Configure data ingestion\n\nConfigure the data ingestion in your `.gitlab-ci.yml` file. This creates raw project metadata shared across all metrics.\n\n#### Basic configuration variables:\n\n```yaml\nvariables:\n  GROUP_PATH: \"gitlab-org/professional-services-automation\"  # Your group path\n  INCLUDE_SIMPLE_FIELDS: \"fullPath,name,description\"  # Fields to fetch\n  ARGUMENT_FIELDS: \"mergeRequests\"  # Argument fields requiring additional config\n  LIMIT: \"100\"  # Projects per API call\n  MAX_ITERATIONS: \"2\"  # Maximum API calls (for testing)\n  GITLAB_GRAPHQL_API_VERSION: \"0.1.0\"  # API version from Package Registry\n```\n\n#### Configure argument fields for specific metrics:\n\nFor each metric you want to collect, define argument field variables:\n\n```yaml\n# Example: Merged Merge Requests\nARGUMENT_FIELD_1_NAME: \"mergeRequests\"\nARGUMENT_FIELD_1_FILTER_NAME: \"state\"\nARGUMENT_FIELD_1_FILTER_VALUE: \"merged\"\nARGUMENT_FIELD_1_RETURN_VALUES: \"count totalTimeToMerge\"\n\n# Example: Packages Count\nARGUMENT_FIELD_2_NAME: \"packages\"\nARGUMENT_FIELD_2_RETURN_VALUES: \"count\"\n```\n\n### 4. Configuring metrics aggregation\n\nAfter data ingestion, set up aggregation rules in `.gitlab/Schedule.gitlab-ci.yml` for each metric you want to generate.\n\n#### Example metric job configuration:\n\n```yaml\nprocess_average_time_to_merge:\n  \u003C\u003C: *process_data_template\n  stage: process_data\n  variables:\n    METRIC_NAME: \"9_Average_Time_To_Merge\"\n    BUSINESS_LEVEL_START: 2\n    BUSINESS_LEVEL_END: 4\n    AGGREGATE_COLUMNS: \"mergeRequests_state_merged_totalTimeToMerge:sum mergeRequests_state_merged_count:sum\"\n    NEW_COLUMN_NAME: \"average_time_to_merge_days\"\n    NEW_COLUMN_FORMULA: \"mergeRequests_state_merged_totalTimeToMerge / mergeRequests_state_merged_count / (24 * 60 * 60)\"\n    SORT_BY: \"average_time_to_merge_days\"\n    FILTER_CONDITION: \"mergeRequests_state_merged_count >= 5\"\n```\n\n    #### Required variables for each metric:\n\n| Variable | Description | Example |\n|----------|-------------|---------|\n| `METRIC_NAME` | Name of the metric (used in file naming) | \"9_Average_Time_To_Merge\" |\n| `BUSINESS_LEVEL_START` | Starting level for business hierarchy | 1 |\n| `BUSINESS_LEVEL_END` | Ending level for business hierarchy | 3 |\n| `AGGREGATE_COLUMNS` | Columns to aggregate with function | \"mergeRequests_state_merged_count:sum\" |\n| `SORT_BY` | Column to sort results by | \"average_time_to_merge_days\" |\n\n\n### 5. Run scheduled pipeline to configure analytics\n\nOnce configured, run a scheduled pipeline to generate your analytics:\n\n1. Go to **CI/CD → Schedules**\n2. Create a new schedule or run an existing one\n3. The pipeline will automatically:\n  - Ingest data based on your configuration\n  - Aggregate metrics according to your rules\n  - Deploy dashboards to GitLab Pages\n\n### 6. Access your analytics dashboards\n\nAfter the scheduled pipeline completes successfully, GitLab Pages automatically deploys your dashboards:\n\n- **Project Metrics Dashboard**: `https://your-username.gitlab.io/project-name/existing-metrics/`\n- **Duo Metrics Dashboard**: `https://your-username.gitlab.io/project-name/duo-metrics/`\n- **Main Landing Page**: `https://your-username.gitlab.io/project-name/`\n\nThe main landing page automatically detects which dashboards are available and displays appropriate links.\n\nYou'll see:\n\n- License utilization: Total licensed users vs. active users (together with code suggestion analytics)\n\n![GitLab Duo Analytics Dasboard](https://res.cloudinary.com/about-gitlab-com/image/upload/v1754478265/nhbukcflhmghs5jatrip.png)\n\n\n- GitLab Duo Chat analytics: Unique Duo Chat users, average Chat events over 90 days, and Chat adoption rate\n- GitLab Duo engagement analytics: Categorizing Duo usage for a group of users as Power (10+ suggestions), Regular (5-9), or Light (1-4) based on usage patterns\n\n![Duo Chat Analytics - last 90 days](https://res.cloudinary.com/about-gitlab-com/image/upload/v1754478265/xgq05hh2ybzb8ugsxqza.png)\n\n- Usage analytics: Code suggestions by programming language (language coverage distribution), Code suggestions language performance analytics (accepted vs rejected rate)\n\n![User adoption view of Duo analytics](https://res.cloudinary.com/about-gitlab-com/image/upload/v1754478265/mu3dx5g2l2lki2ehlr2g.png)\n\n\u003Cp>\u003C/p>\n\n\n![Language performance analytics](https://res.cloudinary.com/about-gitlab-com/image/upload/v1754478267/xf0thn8sm4dlhoyyqg9i.png)\n\n- Weekly Duo Chat trends: Duo Chat usage patterns\n\n![Duo Chat daily usage trends](https://res.cloudinary.com/about-gitlab-com/image/upload/v1754478265/plhycnmewye3vp6vitqj.png) \n\n#### Dashboard features include:\n\n- **License Usage Analytics** - Track licensed vs active users\n- **Code Suggestions Analytics** - Monitor acceptance rates and language distribution\n- **Duo Chat Analytics** - View chat interactions and adoption rates\n- **User Engagement Analytics** - Categorize users by activity level\n- **Language Performance Analytics** - Analyze acceptance rates by programming language\n\n\n### 7. Understanding the APIs used\n\nThe solution leverages multiple GitLab APIs to collect comprehensive usage data:\n\n#### AI usage data API (aiUsageData)\n\n```graphql\n# Fetches individual code suggestion events\nquery: |\n  {\n    group(fullPath: \"your-group\") {\n      aiUsageData {\n        codeSuggestionEvents {\n          event         # ACCEPTED or SHOWN\n          timestamp     # When it happened\n          language      # Programming language\n          suggestionSize # SINGLE_LINE or MULTI_LINE\n          user { username }\n        }\n      }\n    }\n  }\n# Purpose: Tracks every code suggestion shown or accepted by developers\n```\n\n#### GitLab Self-Managed add-on users API\n\n```graphql\n# Gets licensed user information\nquery: |\n  {\n    selfManagedAddOnEligibleUsers(\n      addOnType: DUO_ENTERPRISE\n      filterByAssignedSeat: \"Yes\"\n    ) {\n      user {\n        username\n        lastDuoActivityOn\n      }\n    }\n  }\n# Purpose: Identifies who has licenses and when they last used Duo\n```\n\n#### AI metrics API\n\n```graphql\nquery: |\n  {\n    aiMetrics(from: \"2024-01-01\", to: \"2024-06-30\") {\n      codeSuggestions {\n        shownCount\n        acceptedCount\n      }\n      duoChatContributorsCount\n      duoAssignedUsersCount\n    }\n  }\n# Purpose: Gets pre-calculated metrics for trend analysis\n```\n\n#### Service Ping API (REST)\n\n```bash\nurl: \"{GITLAB_URL}/api/v4/usage_data/service_ping\"\n# Purpose: Collects instance-wide usage statistics\n```\n## Putting it all together\n\nTo demonstrate the power of this integrated analytics solution, let's walk through a complete end-to-end implementation journey — from initial deployment to fully automated ROI measurement.\n\n\nStart by deploying the containerized solution in your environment using the provided Docker configuration. Within minutes, you'll have both the analytics API and React dashboard running locally. \n\n\nThe hybrid data architecture approach immediately begins collecting metrics from your existing monthly CSV exports while establishing real-time GraphQL connections to your GitLab instance.\n\n\n**Automation through Python scripting**\n\n\nThe real power emerges when you leverage Python scripting to automate the entire data collection and processing workflow. The solution includes comprehensive Python scripts that can be easily customized and scheduled.\n\n\n**GitLab CI/CD integration**\n\n\nFor enterprise-scale automation, integrate these Python scripts into scheduled GitLab [CI/CD](https://about.gitlab.com/topics/ci-cd/) pipelines. This approach leverages your existing GitLab infrastructure while ensuring consistent, reliable data collection:\n\n\n```yaml\n\n# .gitlab-ci.yml example\n\nduo_analytics_collection:\n  stage: analytics\n  script:\n    - python scripts/enhanced_duo_data_collection.py\n    - python scripts/metric_aggregations.py\n    - ./deploy_dashboard_updates.sh\n  schedule:\n    - cron: \"0 2 1 * *\"  # Monthly on 1st at 2 AM\n  only:\n    - schedules\n```\n\n\nThis automation strategy transforms manual data collection into a self-sustaining analytics engine. Your Python scripts execute monthly via GitLab pipelines, automatically collecting usage data, calculating ROI metrics, and updating dashboards — all without manual intervention.\n\n\nOnce automated, the solution operates seamlessly: Scheduled pipelines execute Python data collection scripts, process GraphQL responses into business metrics, and update dashboard data stores. You can watch as the dashboard populates with real usage patterns: code suggestion volumes by programming language, user adoption trends across teams, and license utilization rates that reveal optimization opportunities.\n\n\nThe real value emerges when you access the ROI Overview dashboard. Here, you'll see concrete enagagement metrics metrics which can be converted into business impact for your organisation — perhaps discovering that your active Duo users are generating 127% monthly ROI through time savings and productivity gains, while 23% of your licenses remain underutilized. These insights immediately translate into actionable recommendations: expand licenses to high-performing teams, implement targeted training for underutilized users, and build data-driven business cases for broader AI adoption.\n\n\n## Why GitLab?\n\n\nGitLab's comprehensive DevSecOps platform provides the ideal baseline for enterprise AI analytics and measurement. With native GraphQL APIs, flexible data access, and integrated AI capabilities through GitLab Duo, organizations can centralize AI measurement across the entire development lifecycle without disrupting existing workflows.\n\n\nThe solution's open architecture enables custom analytics solutions like the one developed through our Duo Accelerator program. GitLab's commitment to API-first design means you can extract detailed usage data, integrate with existing enterprise systems, and build sophisticated ROI calculations that align with your organization's specific metrics and reporting requirements.\n\n\nBeyond technical capabilities, our approach ensures you're not just implementing tools — you're building sustainable AI adoption strategies. This purpose built solution emerging from the Duo Accelerator program exemplifies this approach, providing hands-on guidance, proven frameworks, and custom solutions that address real enterprise challenges like ROI measurement and license optimization.\n\n\nAs GitLab continues enhancing native analytics capabilities, this foundation becomes even more valuable. The measurement frameworks, KPIs, and data collection processes established through custom analytics solutions seamlessly transition to enhanced native features, ensuring your investment in AI measurement grows with GitLab's evolving solution.\n\n\n## Try GitLab Duo today\n\n\nAI ROI measurement is just the beginning. With GitLab Duo's capabilities you can build out comprehensive analytics. With this you're not just tracking AI usage — you're building a foundation for data-driven AI optimization that scales with your organization's growth and evolves with GitLab's expanding AI capabilities.\n\n\nThe analytics solution developed through GitLab's Duo Accelerator program demonstrates how customer success partnerships can deliver immediate value while establishing long-term strategic advantages. From initial deployment to enterprise-scale ROI measurement, this solution provides the visibility and insights needed to maximize AI investments and drive sustainable adoption.\n\n\nThe combination of Python automation, GitLab CI/CD integration, and purpose-built analytics creates a competitive advantage that extends far beyond individual developer productivity. It enables strategic decision-making, optimizes resource allocation, and builds compelling business cases for continued AI investment and expansion.\n\n\nThe future of AI-powered development is data-driven, and it starts with measurement. Whether you're beginning your AI journey or optimizing existing investments, GitLab provides both the platform and the partnership needed to succeed.\n\n> Get started with GitLab Duo today with a [free trial of GitLab Ultimate with Duo Enterprise](https://about.gitlab.com/gitlab-duo/).\n",[720,694,9],"2025-08-06","2025-09-09",{"featured":6,"template":700,"slug":3360},"measuring-ai-roi-at-scale-a-practical-guide-to-gitlab-duo-analytics","content:en-us:blog:measuring-ai-roi-at-scale-a-practical-guide-to-gitlab-duo-analytics.yml","Measuring Ai Roi At Scale A Practical Guide To Gitlab Duo Analytics","en-us/blog/measuring-ai-roi-at-scale-a-practical-guide-to-gitlab-duo-analytics.yml","en-us/blog/measuring-ai-roi-at-scale-a-practical-guide-to-gitlab-duo-analytics",{"_path":3366,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3367,"content":3373,"config":3378,"_id":3380,"_type":13,"title":3381,"_source":15,"_file":3382,"_stem":3383,"_extension":18},"/en-us/blog/memory-safe-vs-unsafe",{"title":3368,"description":3369,"ogTitle":3368,"ogDescription":3369,"noIndex":6,"ogImage":3370,"ogUrl":3371,"ogSiteName":685,"ogType":686,"canonicalUrls":3371,"schema":3372},"How to secure memory-safe vs. manually managed languages","Learn how GitLab reduces source code risk using scanning, vulnerability management, and other key features.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749672878/Blog/Hero%20Images/securityscreen.jpg","https://about.gitlab.com/blog/memory-safe-vs-unsafe","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to secure memory-safe vs. manually managed languages\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2023-03-14\",\n      }",{"title":3368,"description":3369,"authors":3374,"heroImage":3370,"date":3375,"body":3376,"category":830,"tags":3377},[1444],"2023-03-14","The National Security Agency (NSA) has published an executive summary\nshowcasing the \n\nrisk of using [manually managed\nlanguages](https://en.wikipedia.org/wiki/Manual_memory_management) over\n[memory-safe\nlanguages](https://en.wikipedia.org/wiki/Garbage_collection_(computer_science))\nin application\n\ndevelopment. Manual memory management may introduce major bugs and security\nrisks into your application if\n\nthe memory is managed incorrectly.\n\n\nSecurity bugs introduced by manually managed languages can be catastrophic\nto the function of the\n\napplication, as well as the information contained in the application. These\nbugs may cause\n\nperformance slowdowns, application crashes, remote code execution,\ninformation leakage, and\n\nsystem failures.\n\n\nBugs that may be introduced include the following:\n\n\n* [Memory leak](https://en.wikipedia.org/wiki/Memory_leak): Memory no longer\nbeing used is not released, which reduces the amount of available memory.\n\n* [Buffer overflow](https://en.wikipedia.org/wiki/Buffer_overflow):\nOverwriting of memory locations adjacent to a buffers boundary.\n\n* [Segmentation fault](https://en.wikipedia.org/wiki/Segmentation_fault): An\napplication tries to access a restricted piece of memory.\n\n* [Wild pointers](https://en.wikipedia.org/wiki/Dangling_pointer): Pointer\npoints to the memory which has been deallocated.\n\n* [Undefined behavior](https://en.wikipedia.org/wiki/Undefined_behavior): An\napplication with unpredictable behavior.\n\n\nTo provide some insight on the prevalance of risk introduced by manually\nmanaged languages, Microsoft\n\nrevealed that within the span of 12 years, [70% of their\nvulnerabilities](https://github.com/Microsoft/MSRC-Security-Research/blob/master/presentations/2019_02_BlueHatIL/2019_01%20-%20BlueHatIL%20-%20Trends%2C%20challenge%2C%20and%20shifts%20in%20software%20vulnerability%20mitigation.pdf)\nwere due to mismanagement of memory.\n\nGoogle reported that there was a [similar percentage of\nvulnerabilities](https://security.googleblog.com/2021/09/an-update-on-memory-safety-in-chrome.html)\nintroduced by memory safety\n\nviolations within the Chrome browser.\n\n\nAll these vulnerabilities can be exploited by malicious actors who may\ncompromise a device, potentially leading to a compromise of a larger network\ninfrastructure. With this large risk presented by mismanaged memory,\n\nthe NSA advises organizations to consider using memory-safe languages\nwherever possible and providing\n\nmechanisms to harden applications built with manually managed languages.\n\n\n## Memory-safe languages vs. manually managed languages\n\n\nA memory-safe language is a language where memory allocation and garbage\ncollection are abstracted away from\n\nthe developer and handled by the programming language itself. These\nlanguages include **Python**, **Java**, and **Go**,\n\nto name a few.\n\n\nIn contrast, manually managed languages provide a developer with full\ncontrol over the system memory (with some exceptions).\n\nThe most popular manually managed languages are **C** and **C++**.\n\n\nEach language type has a purpose and use case. There are times when a\nmemory-safe language is recommended, but there are also\n\ntimes when it may not suit the application requirements.\n\n\nBelow is a list of some pros and cons of each language type:\n\n\n| Language type | Pros | Cons |\n\n| ------------- | ---- | ---- |\n\n| Memory safe | Memory mangement abstracted from developer, reduced risk of\nmemory errors | Reduced efficency/performance, unpredictable garbage\ncollection |\n\n| Manually managed | Enhanced efficency/performance, no garbage collection\noverhead | Prone to memory-related failures |\n\n\nManually managed languages provide the developer with more power, but also\nintroduce a greater amount\n\nof risk, so they should only be used where required.\n\n\n## Memory 'unsafe' language security scanning\n\n\nAlthough many organizations are promoting the use of memory-safe languages\nvs. manually managed ones, it is unrealistic\n\nto remove manually managed languages from a developer's toolbox. Therefore,\ndevelopers must get ahead of all the\n\nbugs/vulnerabilities that may be introduced. This can be done by scanning\napplication source code.\n\n\nGitLab supports various scanners for memory-unsafe languages. Below you can\nsee the scanners\n\nused for C and C++:\n\n\n| Language | Scanners |\n\n| -------- | -------- |\n\n| C | [Semgrep with GitLab-managed\nrules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) |\n\n| C++ |\n[Flawfinder](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder)\n|\n\n\nNow let's take a look at how [GitLab's static application security testing\n(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/) allows\nus to find and resolve vulnerabilities.\n\nBelow is an application which doesn't crash, but may generate unexpected\nbehavior:\n\n\n```C\n\n#include \u003Cstdio.h>\n\n\nint main()\n\n{  \n  char msg[5] = \"Hello\";\n\n  /* Add exclamation, to a position that doesn't exist*/\n  msg[8] = '!';\n\n  /* print each letter 1 by 1 */\n  /* Notice we are going further than the length of the array */\n  int i;\n  for (i = 0; i \u003C 10; ++i)\n  {\n    printf(\"%i: %c \\n\", i, msg[i]);\n  }\n\n  return 0;\n}\n\n```\n\n\nWhen running the GitLab SAST scanner, the vulnerability is detected and a\nsolution is provided:\n\n\n![GitLab SAST scanner\nresults](https://about.gitlab.com/images/blogimages/memory-safe-vs-manually-managed/CWE_120.png)\n\n\nIt shows you need to **perform bounds checking, use functions that limit\nlength**, or\n\n**ensure that the size is larger than the maximum possible length.** You can\nalso see the\n\n[CWE](https://cwe.mitre.org/data/definitions/120.html) for more information\non how the system may be impacted.\n\nNote that vulnerabilities are actionable. These actions include the ability\nto dismiss a vulnerability and add\n\nadditional information for the security team to review, or a confidential\nissue can be created for review.\n\n\nThese scanners allow [DevSecOps](/topics/devsecops/) teams to resolve\nsecurity issues before code makes it into production and safeguard their\napplication\n\nfrom memory issues. Note that not all memory issues are easily detected due\nto the nature of manual memory management.\n\nTherefore, it is also important to add unit tests, fuzzing, and run checks\nusing the GitLab CI to further ensure the reliability\n\nand security of your application.\n\n\nThe following applications contain examples of creating a GitLab pipeline\nfor C applications:\n\n- [General Build, SAST, and\nRun](https://gitlab.com/tech-marketing/devsecops/initech/other/cul8r)\n\n- [Coverage-based\nfuzzing](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/c-cpp-fuzzing-example)\n\n\n## Memory-safe language security scanning\n\n\nAs more developers move to memory-safe languages, it is important that the\ntools\n\nused to prevent vulnerabilities support these languages as well. GitLab\nprovides a rich feature set for\n\nsecuring application source code, especially for memory-safe languages.\n\n\nBelow is a table of some the popular languages GitLab supports. To see the\nfull list, visit the\n\n[GitLab SAST Language/Framework\nSupport](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks)\npage.\n\n\n| Language | Scanners |\n\n| -------- | -------- |\n\n| Python | [Semgrep with GitLab-managed\nrules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep),\n[Bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) |\n\n| Go | [Semgrep with GitLab-managed\nrules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep),\n[GoSec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) |\n\n| Java | [Semgrep with GitLab-managed\nrules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep),\n[SpotBugs with the find-sec-bugs\nplugin](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs),\n[MobSF\n(beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) |\n\n| JavaScript | [Semgrep with GitLab-managed\nrules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep),\n[ESLint security\nplugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) |\n\n| Ruby |\n[brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman)\n|\n\n\nGitLab uses a mix of open source tools developed in-house as well as\ncommonly used tools within the open source community.\n\nIt is important to note that GitLab's security research team creates custom\nrules to better reduce false positives as well\n\nas enhance the number of vulnerabilities found.\n\n\nHere are some Python functions, which can be exploited and then data can be\nobtained via [SQL\ninjection](https://owasp.org/www-community/attacks/SQL_Injection):\n\n\n```python\n\ndef select_note_by_id(conn, id=None, admin=False):\n   query = \"SELECT id, data FROM notes WHERE secret IS FALSE\"\n   cur = conn.cursor()\n\n   # Admin doesn't have search by id function, since only used in the UI\n   if admin:\n       query = \"SELECT id, data, ipaddress, hostname, secret FROM notes\"\n\n   if id:\n       if admin:\n           query = query + \" WHERE id = %s\" % id\n       else:\n           # NOTE: Vulnerable to SQL injection, can get secret notes\n           # by adding 'OR 1=1', since not parameterized\n           query = query + \" AND id = %s\" % id\n\n   try:\n       cur.execute(query)\n   except Exception as e:\n       note.logger.error(\"Error: cannot select note by id - %s\" % e)\n\n   allItems = cur.fetchall()\n   conn.close()\n\n   if len(allItems) == 0:\n       return []\n\n   return allItems\n```\n\n\nWhen running the GitLab SAST scanner, you can see the SQL injection\nvulnerability is detected. A solution\n\nis provided with the line of code affected as well as identifiers that\nprovide more information on how the\n\n[CWE](https://cwe.mitre.org/data/definitions/89.html) can affect your\nsystem.\n\n\n![SQL Injection and\nsolution](https://about.gitlab.com/images/blogimages/memory-safe-vs-manually-managed/CWE_89.png)\n\n\nNotice that there is also training to enable developers to understand the\nvulnerability and how\n\nit can be exploited, and to make them more security-aware.\n\n\n## Other application attack vectors\n\n\nUsing a memory-safe language along with a SAST scanner reduces vulnerability\nrisk, but there are more attack vectors to consider, including\nconfigurations, infrastructure, and dependencies. This is why it is\nimportant to scan all aspects of your application.\n\n\nGitLab offers the following scanners to help you achieve full coverage:\n\n\n| Scanner type | Description |\n\n| ------------ | ----------- |\n\n| [Dynamic application security testing\n(DAST)](https://docs.gitlab.com/ee/user/application_security/dast/) |\nExamines applications for vulnerabilities like these in deployed\nenvironments. |\n\n| [Infrastructure as code (IaC)\nscanning](https://docs.gitlab.com/ee/user/application_security/iac_scanning/)\n| Scans your IaC (Terraform, Ansible, AWS CloudFormation, Kubernetes, etc.)\nconfiguration files for known vulnerabilities. |\n\n| [Dependency\nscanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/)\n| Finds security vulnerabilities in your software dependencies. |\n\n| [Container\nscanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/)\n| Scans your applications container images for known vulnerabilities. |\n\n| [License scanning -\nCycloneDX](https://docs.gitlab.com/ee/user/compliance/license_scanning_of_cyclonedx_files/index.html)\n| Capable of parsing and identifying over 500 different types of licenses\nand can extract license information from packages that are dual-licensed or\nhave multiple different licenses that apply. |\n\n| [Secret\ndetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/)\n| Scans your repository for secrets. |\n\n| [Coverage-guided\nfuzzing](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/)\n| Sends random inputs to an instrumented version of your application in an\neffort to cause unexpected behavior. |\n\n| [Web API\nfuzzing](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/)\n| Sets operation parameters to unexpected values in an effort to cause\nunexpected behavior and errors in the API backend. |\n\n\nAside from full scanner coverage, it important to add guardrails to prevent\nvulnerable code from compromising a production environment.\n\nThis can be done by requiring approval from the security team for merging\nany code with vulnerabilities using [GitLab\npolicies](https://docs.gitlab.com/ee/user/application_security/policies/).\n\n\nTo get started using these tools and more, check out the [GitLab Application\nSecurity](https://docs.gitlab.com/ee/user/application_security/) page.\n\nIt's as simple as signing up for GitLab Ultimate and adding some templates\nto your .gitlab-ci.yml.\n\n\n## Managing vulnerabilities of all types\n\n\nAlthough we can find and address vulnerabilities before they make it into\nproduction, it is not possible to\n\neliminate all risk. This is why it is important to be able to assess the\nsecurity posture of your project or\n\ngroup of projects.\n\n\nFor this, GitLab provides [Vulnerability\nReports](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/),\nwhich allow you to manage and triage vulnerabilities\n\nwithin the main branch of the application. You can sort through all the\nvulnerabilities for a project or\n\ngroup of projects using a variety of different criteria.\n\n\n![Vulnerability report\nscreenshot](https://about.gitlab.com/images/blogimages/memory-safe-vs-manually-managed/vulnerability_report.png)\n\n\nClicking on a vulnerability sends you to its [Vulnerability\nPage](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/).\n\nThere you can review details on the vulnerability, manage its status,\ncollaborate with other members of\n\nthe security team, as well as create confidential issues to assign to\ndevelopers.\n\n\n![Vulnerability\nPage](https://about.gitlab.com/images/blogimages/memory-safe-vs-manually-managed/vulnerability_page.png)\n\n\n---\n\n\nThanks for reading! To learn more about available Security features, check\nout GitLab's [application security\ndocumentation](https://docs.gitlab.com/ee/user/application_security/)\n\nand get started securing your application today. You can also sign up for a\n[free GitLab Ultimate trial](https://about.gitlab.com/free-trial/)\n\nand test the [Simple Notes\nApplication](https://gitlab.com/tech-marketing/devsecops/initech/simple-notes),\nwhich contains a [full\ntutorial](https://tech-marketing.gitlab.io/devsecops/initech/simple-notes/)\non getting started\n\nwith implementing and using many of GitLab's security features.\n\n\n## References\n\n\nBelow are some references used in this blog:\n\n\n* [The Federal Government is Moving on Memory Safety for\nCybersecurity](https://www.nextgov.com/cybersecurity/2022/12/federal-government-moving-memory-safety-cybersecurity/381275/)\n\n* [Future of Memory Safety - Challenges and\nRecommendations](https://advocacy.consumerreports.org/wp-content/uploads/2023/01/Memory-Safety-Convening-Report-1-1.pdf)\n\n* [NSA Software Memory Safety\nReport](https://media.defense.gov/2022/Nov/10/2003112742/-1/-1/0/CSI_SOFTWARE_MEMORY_SAFETY.PDF)\n\n* [Memory Safety Wiki](https://en.wikipedia.org/wiki/Memory_safety)\n\n* [Manual Memory Management\nWiki](https://en.wikipedia.org/wiki/Manual_memory_management)\n\n* [Unsafe Languages - University of Washington CS Lecture\nNotes](https://courses.cs.washington.edu/courses/cse341/04wi/lectures/26-unsafe-languages.html)\n\n* [GitLab SAST](https://docs.gitlab.com/ee/user/application_security/sast/)\n\n* [GitLab Application\nSecurity](https://docs.gitlab.com/ee/user/application_security/)\n\n* [GitLab Vulnerability\nReports](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/)\n\n\n_Cover image by [Mohammad\nRahmani](https://unsplash.com/@afgprogrammer?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)\non\n[Unsplash](https://unsplash.com/s/photos/C-programming?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)_\n",[830,851,9],{"slug":3379,"featured":6,"template":700},"memory-safe-vs-unsafe","content:en-us:blog:memory-safe-vs-unsafe.yml","Memory Safe Vs Unsafe","en-us/blog/memory-safe-vs-unsafe.yml","en-us/blog/memory-safe-vs-unsafe",{"_path":3385,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3386,"content":3391,"config":3397,"_id":3399,"_type":13,"title":3400,"_source":15,"_file":3401,"_stem":3402,"_extension":18},"/en-us/blog/migration-from-atlassian-bamboo-server-to-gitlab-ci",{"title":3387,"description":3388,"ogTitle":3387,"ogDescription":3388,"noIndex":6,"ogImage":2238,"ogUrl":3389,"ogSiteName":685,"ogType":686,"canonicalUrls":3389,"schema":3390},"Migrating from Bamboo Server to GitLab CI: Getting started","Theoretical reasoning and practical proposal on migrating an existing CI/CD infrastructure of some multi-component application from Bamboo Server to GitLab CI","https://about.gitlab.com/blog/migration-from-atlassian-bamboo-server-to-gitlab-ci","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to migrate Atlassian Bamboo Server's CI/CD infrastructure to GitLab CI, part one\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ivan Lychev\"}],\n        \"datePublished\": \"2022-07-06\",\n      }",{"title":3392,"description":3388,"authors":3393,"heroImage":2238,"date":3394,"body":3395,"category":785,"tags":3396},"How to migrate Atlassian Bamboo Server's CI/CD infrastructure to GitLab CI, part one",[2778],"2022-07-06","\n\nWhen I faced a task of migrating from `Atlassian Bamboo Server` to `GitLab CI/CD`, I was not able to find any comprehensive information regarding something similar. So I designed a process on my own. This demo shows how to migrate a CI/CD structure for an existing multi-component application from a discontinued [Atlassian Bamboo Server](https://www.atlassian.com/migration/assess/journey-to-cloud) to [GitLab CI/CD](https://docs.gitlab.com/ee/index.html) (Community Edition).\n\nThe accompanying repository is https://gitlab.com/iLychevAD/ci-cd-for-a-multi-component-app.\n\nIn this first part of a two-part series, you will find a description of the current state of affairs - i.e., how the CI/CD has been organized within Bamboo Server, how the Bamboo Build and Deploy plans are designed for bootstrapping infrastructure and deploying the components of the application, and the architecture of the application itself.\n\nAnd in part two, we'll take a deeper look at the virtues of `GitLab CI/CD`.\n\n## Initial state\n\n(Note: This is not a description of some particular project but more a kind of compilation of several projects I worked on.)\n\nThe application solution allows the client to fulfill a particular business purpose (the nature of which is not relevant here and thus not specified) and consists of more than 50 discrete components (further referred to as `applications` or just `apps` or `components`). I refrain from calling them microservices as each of them looks more like a full-fledged application communicating with other siblings using REST API and messages in Kafka topics. Some of them expose a web UI to external or internal users and some are just utility parts serving the needs of other components or performing internal operations, etc.\n\nCode for each app is stored in its own Git repository (further just `repo`). So, a `multi-repo` approach is used for them. Each app may be written in different languages and packaged as one or several OCI-images for deployment.\n\nEach app repo looks like:\n```\n📦 \u003Csome-app-git-repo>\n ┣ 📂src \u003C-- application source code\n ┣ 📂docker-compose\n ┃ ┗ 📜docker-compose.yml \u003C-- analogue of K8s manifests\n ┗ 📜Dockerfile \u003C-- conventionally, \"Dockerfile\" name is used for OCI image specification file\n```\n\nFor running the applications, the client uses an outdated orchestration system (one from pre-Kubernetes epoch). So each app repo contains a Docker-compose compatible file describing deployment directives for that outdated orchestration system (in essence, similar to Kubernetes Deployment manifests). \n\nFor all of the build and deploy activities Atlassian Bamboo Server is used. \n\nSome details for those not familiar with the Bamboo Server - in an opinionated manner it explicitly separates so-called `build` pipelines and `deployment` pipelines. The former are supposed to build application code and produce some artifacts for further deployment (in our case those artifacts are OCI images uploaded to OCI registry and docker-compose.yml files referring to those images). The latter ones are supposed to take some particular set of artifacts and apply them to some particular `environment`. An `environment` (referred to `env` in the future for brevity) here is just an abstract deployment target characterized by a set of environment variables attached to it and exposed to the apps deployed into it. In reality, an `env` is implemented as a set of resources (virtual machines, databases, object storage locations, etc.) required by the applications.\n\nIn Bamboo, one `build` pipeline usually corresponds to one `deployment` pipeline so when the latter is started it just takes the artifacts from the attached `build` pipeline as input. \n\nThe client uses a `production` env, `preproduction` env, and numerous (up to several hundreds) so-called `staging` (short-lived) envs where different development teams and software engineers can test various combinations of the apps (here we assume that they have ~80-100 distinguish components of the application solution and several hundreds of software developers which gives a lot of possible combinations and requires so many `staging` envs).\n\nRoughly, a configuration of a `deploy` pipeline consists of a specification of the source artifacts (which are provided by the attached `build` pipeline as described earlier) and a specification of the set of envs where those artifacts (effectively, an application) can be deployed to.\n\nCurrent installation uses sophisticated dynamic generation of envs set for each app deployment pipeline. Roughly speaking, they have a central configuration file with the list of all existing envs where for each env a list of apps allowed to be deployed to it is denoted. Each time the file is modified (i.e., an env is created or deleted), the deployment pipelines are automatically being updated so as in the result each of them contains a list of envs corresponding for each app. You will have more idea about this aspect when you have looked at the implementation section later.\n\nIn the Bamboo UI this looks like:\n\n![envs_list_on_build_result_page](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/envs_list_on_build_result_page.png)\n\nHere you can see an application build result page where on the right-hand side under the `Included in deployment project` title you can see a list of envs into which you can deploy the application. (Keep in mind that besides `build` and `deployment` pipelines, the Bamboo also uses a notion of `releases` - this is just some kind of an intermediate entity that should be created out of a build result to make it possible to deploy that build into some env). The `cloud-with-upwards-arrow` button in the `Actions` column starts a corresponding `deploy` pipeline with automatically passing the link to a build result (in a form of a `release` entity in Bamboo terminology) and the name of the env next to which the button has been clicked (the procedure of how a list of envs is created for a `deploy` pipe is described above).\n\nA concept of a `release` is specific to Bamboo Server, though it provides some amenities. For example, on the Release details page you can see a list of envs where a release has been deployed to. On the `Commits` tab you can backtrack a release to the application code in a SVC. And the `Issues` tab shows attached Jira tickets.\n\n![bamboo_release_details](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/bamboo_release_details.png)\nRelease details page\n{: .note.text-center}\n\nAn env details page also enumerates releases history for this env (in scope of one particular application though as an env is specified for each deployment pipeline individually):\n\n![bamboo_env_details](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/bamboo_env_details.png)\nEnv details page \n{: .note.text-center}\n\nAnd upon clicking the `cloud-with-upwards-arrow` button the Bamboo shows diff of Jira tickets and commits in respect to the previous `release` (only if both releases are made from artifacts from the same Git branch):\n\n![deploy_launch_page](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/deploy_launch_page.png)\nDeploy launch page\n{: .note.text-center}\n\nSo, in general, the current path from source control to an env for each app looks like:\n\n![svc_to_env_path](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/svc_to_env_path.png)\n\nThe Build plans are triggered automatically upon Git commits or Git tags. Most of the Deployment plans are started by the project members manually when needed. Each Deploy plan contains a step that checks if a user who started the plan has permissions to deploy into an env (for example, only members of the team which owns an env are allowed to deploy to that env and the deployment to the production env is allowed only for a set of eligible project members).\n\n## The task\n\nThe task is to migrate the aforementioned design from Bamboo Server to `GitLab` while keeping a similar deployment scheme (leveraging GitLab's `Environments` feature).\n\nAlso the following should be considered:\n\n - team members (software engineers, quality assurance specialists) are supposed to be able to manage environments on their own in a user-friendly self-service manner.\n - there should not be any discrepancy in IaC for different environments (per `12-factor apps` best practices), i.e. for any kind of an environment, be it a development or production one, the same set of IaC (here - Terraform files) should be used.\n  - the core ideas and workflows established in the previous situation (implemented with Atlassian Bamboo) should be kept to make the migration smoother for the members of the projects (also sometimes referred to as just users). \n\n## Implementation\n\n### Implementation's GitLab groups\\projects structure\n\n```\n📦 \u003CGitLab root group>\n ┣ 📂 apps GitLab group\n ┃ ┣ 📃 app1 GitLab project\n ┃ ┣  ...\n ┃ ┗ 📃 appN GitLab project\n ┣ 📂 ci GitLab group\n ┃ ┣ 📃 library GitLab project\n ┃ ┗ 📃 oci-registry GitLab project\n ┗ 📂 infra GitLab group\n  ┣ 📃 environment-blueprints GitLab project\n  ┣ 📃 environment-set GitLab project\n  ┗ 📃 k8s-gitops GitLab project\n```\n\n*Description*:\n\nThe most important content is in the `ci/library` repo (the shared ci configs) and `environment-set` repo. The other repos don't require much attention: The `k8s-gitops` purpose is not implemented and the repo is empty, the `apps` group just imitates source code for some apps, and the `ci/oci-registry` serves a role of an OCI registry for the solution.\n\nThe `apps` GitLab group merely contains the apps source code per se. Each GitLab project in this group corresponds to one app. Each app repo is expected to contain the source code itself (in the `src` directory for example), a `k8s` directory with k8s manifests, and an OCI image specification file (traditionally often called `Dockerfile`). \n\nThe `ci` GitLab group contains the `ci/library` project that holds shared `.gitlab-ci.yaml` files used by other projects (in a manner similar to Jenkins' shared libraries) and the `ci/oci-registry` serves as an OCI-image registry for various images used by the demo project (it also contains a Git repository with gitlab-ci files to build some utility images with tools used in various pipelines). For simplicity, the latter stores all the images throughout all the projects of the demo, though it's clearly not the best choice for a real-life situation when different sets of images of a set of separate projects/registries should be created.\n\nThe `infra` group holds applications infrastructure creation related Git repositories:\n\nThe `infra/k8s-gitops` is mostly irrelevant to the topic of this demo. In this demo it's presumed that Kubernetes is used as a computation workload platform and when a k8s cluster is created for an environment all the k8s manifests are supposed to be put into this repo (where each branch corresponds to a single environment) to be consumed by a GitOps tool installed into the cluster.\n\nThe `infra/environment-blueprints` holds parametrized IaC templates describing all the resources required for a full-fleged environment. In this example, the Terraform is used as an IaC tool though the principles are similar for its analogs (CloudFormation, for instance). The blueprints are parametrized in such manner that in the defaults values they hold some sensible values (most likely set to different values depending on the kind of a environment they were used to bootstrap - for example, a production env and everything else). It's implied that there might coexist several versions of the blueprints (implemented by using Git branches or Git tags) so each environment (see the next paragraph about `infra/environment-set`) can explicitly specify which version it wants to use (in case of using Terraform by specifying Git reference in the module's `source` field).\n\nHere I would like once again to highlight a digression from the best practices. For simplicity in the `infra/environment-blueprints` repo all the parts of an environment are combined into one single Terraform module (or a workspace, or a Stack in CloudFormation's terminology). In that way all the resources are always updated or changed within a single `terraform apply` command, which is cumbersome for large infrastructures containing a lot of resources. For larger infrastructures it would be more manageable to split into disparate Terraform modules (or CloudFormation Stacks, or Azure ARM Resource Groups) and thus make it possible for the infrastructure to be changed/updated in parts according to which exact components of it have changed. This might raise another question - how to manage dependencies in between such parts if they are present? For that, we would use some kind of an external (in respect to the IaC tool itself) orchestration tool like AWS Step Functions... or even GitLab's DAG feature!\n\nFinally, the `infra/environment-set` project represents an actual expected state of resources for each environment (a branch corresponds to an environment). See the README.md file in the Git repo for details. In short, each branch here is meant to contain a `main.tf` file referring to some version of the blueprints in the `infra/environment-blueprints` project, a set of Terraform files with overrides for any default variables set in the blueprints modules and other utility files like with a list of users allowed to deploy to the environment (such a list is to be checked by the deployments job in the apps projects).\n\n### **Important!**\n\nWhile looking at the implementation keep im mind that this solution deliberately omits some crucial aspects of any project infrastructure like security or monitoring, just for the sake of keeping this solution manageable and comprehensible. Implementing security and monitoring aspects would make the solution cumbersome and much longer to prepare. That is also true for the `k8s-gitops` repository - it's implied that in a real-life solution this would actively participate in the deployment process and hold Kubernetes clusters state in a GitOps approach but currently, this repo is just a placeholder. In the practical guide later you will see a description of the process of controlling environments using different branches in the `infra/environment-set` project. Ideally, such a workflow should use Merge Requests though for simplicity this implementation skips using MRs.\n\nAnother important thing that's possible not clear in this solution is configuration management, i.e. how configuration settings unique to each environment are provided to the applications inside an environment. Well, given that our applications run within Kubernetes cluster and that the cluster state is placed into a dedicated repo (`k8s-gitops` in our case), the configuration settings situation is simple - for each app the Terraform files in the `infra/environment-blueprints` should output all the sensible configuration values for the resources (like S3 bucket names, RDS endpoint URLs, etc.). Then, using Terraform itself or some other tool to create/update an environment, an additional step would collect all those outputs, transform them into k8s ConfigMap manifests, and put them into the GitOps repo. \n\nFor the secrets, we can go several ways. The most simplistic (though not flexible and not easy for secret rotation) way is to use some kind of encryption at rest like Mozilla's SOPS so that the secrets are being encrypted when they are put into the GitOps repo and decrypted when deployed into K8s. Another (and better ?) way - do not store secrets at rest at all but use either a third-party tool like Hashicorp Vault (with dynamic secrets generation) or cloud native features like [AWS IAM Roles for Service Accounts](https://aws.amazon.com/blogs/containers/diving-into-iam-roles-for-service-accounts/).\n\n## Bootstrap the demo\n\nThe accompanying repository, https://gitlab.com/iLychevAD/ci-cd-for-a-multi-component-app, contains Terraform files that enable you to install a copy of the demo structure into your own GitLab account to see it in action:\n\n`*.tf` files in the root directory and in the `tf_modules` directory describe the structure and configuration of the GitLab projects and groups. In the `repo_content` directory there is a content for the GitLab repositories in the projects. The repositories are filled with those files by the Terraform scripts.\n\nThe demo was tested with GitLab Community Edition `15.0.0-pre revision 4bda1cc84df`. The Terraform scripts do not create any real resources but just imitate them using `null_resource` and `local-exec`.\n\nThe bootstrapping process is conducted inside a container image (see the steps below) so it's platform-agnostic and in terms of tools all you need to spin up the demo is some containerization engine installed on your PC (i.e., Docker, Podman, etc).\n\n**Steps**:\n\n1. In the GitLab web UI manually create a root group to bootstrap the demo into (see `root_gitlab_group.tf` for a web-link why it's not possible to automate). Notice its ID - you need to provide it at the next step.\n\n2. Clone this repository.\n    Download an official Hashicorp's Terraform image and enter its interactive shell. All the further commands are supposed to be performed inside that shell:\n    \n    ```\n    docker run --rm -it --name ci-cd-for-a-multi-component-app \\\n      -e TF_VAR_gitlab_token=\u003Cyour GitLab account access token> \\\n      -v \u003Cpath to a location where to store ssh key-pairs on your PC>:/deploy-keys \\\n      -e TF_VAR_deploy_key_readwrite=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key.pub \\\n      -e TF_VAR_deploy_key_readonly=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key.pub \\\n      -e TF_VAR_root_gitlab_group_id=\u003CGitLab group ID> \\\n      -v \u003Cpath to the directory where you cloned the project into>:/repo -w /repo \\\n      --entrypoint /bin/sh \\\n      public.ecr.aws/hashicorp/terraform:1.1.9\n    ```\n    \n    Explanation:\n    \n    `-e TF_VAR_gitlab_token=\u003Cyour GitLab account access token>` - Terraform's `gitlab` provider needs a GitLab access token with sufficient permissions to spin up the demo. Provide it as a Bash environment variable - `TF_VAR_gitlab_token` (see `provider.tf`). It is also used by the `upload_avatar` module.\n    \n    `-v \u003Cpath to a location where to store ssh key-pairs on your PC>:/deploy-keys` - on the left-hand side here specify some directory on your local PC where you would like to store SSH keys needed for deploying the demo. Thus they are persisted even if you exit the container. See bullet point `4` for more details.\n    \n    `-e TF_VAR_deploy_key_readwrite=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key` and\n    \n    `-e TF_VAR_deploy_key_readonly=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key` - set the names for the aforementioned keys\n    \n    `-v \u003Cpath to the directory where you cloned the project into>:/repo -w /repo` - we mount the project content from your local PC into the running container. Note that because of that the Terraform local state file will be stored inside that directory on your PC.\n\n3. Install tools - bash and curl:\n    \n    ```\n    apk add bash curl\n \n    /bin/bash\n    ```\n\n4. Upon bootstrapping the demo, the repositories' content is pushed into (i.e. is restored) from the `repo_content` directory. (When the demo is destroyed the content of the repositories is automatically pulled (i.e. is saved) into the same directory - probably you dont need this but I implemented that for my convinience during creating the demo.) We need to create an SSH key pair and need it be the same throughout both phases. In this step we generate it:\n    \n    ```ssh-keygen -t rsa -N '' -f /deploy-keys/ci-cd-for-a-multi-component-app-deploy-key \u003C\u003C\u003C y```\n    \n    ```chmod 0400 /deploy-keys/ci-cd-for-a-multi-component-app-deploy-key```\n    \n    A trick used in `tf_modules/gitlab_project_with_restore_backup/main.tf` requires that in the host section of the SSH public key the location of the private key is specified (in a form like `filename@~/.ssh/\u003Cfilename>`). Otherwise the `tf_modules/gitlab_project_with_restore_backup` won't work. Edit accordingly:\n    \n    ```sed -i -e 's|^\\(ssh-rsa .*\\) \\(.*\\)$|\\1 ci-cd-for-a-multi-component-app-deploy-key@/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key|' /deploy-keys/ci-cd-for-a-multi-component-app-deploy-key.pub```\n    ```\n\nNow you can proceed with bootstrapping the demo using Terraform:\n\nInitialize Terraform by `terraform init` so it installs all the providers.\n\nDeploy the demo with Terraform by `terraform apply`.\n\n**Notice**: During Terraform execution you may see an error:\n```\nError: POST https://gitlab.com/api/v4/projects/multi-component-app-root-group/ci/library/deploy_keys: 400 {message: {deploy_key.fingerprint_sha256: [has already been taken]}}\n\n```\nI believe this is some glitch in the GitLab API. To fix just run `terraform apply` once again until it shows no errors.\n\nAfter that you should see the following structure in GitLab in the root group:\n\n![gitlab_projects_tree](https://about.gitlab.com/images/blogimages/migration-from-atlassian-bamboo-server-to-gitlab-ci/gitlab_projects_tree.png)\n\nAll the projects should be filled with files from the `repo_content` directory.\n\nDo not delete the directory with the cloned project and the files created inside it if later you would want to clean up the things. See the next section for instructions.\n\n## Cleaning up\n\nLaunch a container image the same way you did for bootstrapping the demo (see the previous section). It's supposed that you didnt delete any files in `\u003Cpath to a location where to store ssh key-pairs on your PC>` and `\u003Cpath to the direcory where you cloned the project into>`: \n\n```\ndocker run --rm -it --name ci-cd-for-a-multi-component-app \\\n  -e TF_VAR_gitlab_token=\u003Cyour GitLab account access token> \\\n  -v \u003Cpath to a location where to store ssh key-pairs on your PC>:/deploy-keys \\\n  -e TF_VAR_deploy_key_readwrite=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key.pub \\\n  -e TF_VAR_deploy_key_readonly=/deploy-keys/ci-cd-for-a-multi-component-app-deploy-key.pub \\\n  -e TF_VAR_root_gitlab_group_id=\u003CGitLab group ID> \\\n  -v \u003Cpath to the direcory where you cloned the project into>:/repo -w /repo \\\n  --entrypoint /bin/sh \\\n  public.ecr.aws/hashicorp/terraform:1.1.9\n```\n\nInstall curl:\n\n```apk add curl```\n\nDo `terraform destroy`.\n\n**Notice**: You may see some errors regarding deleting the `oci-registry` project with OCI images. In that case just delete the images and remove the project manually or wait while GitLab does that itself later.\n\nNow if you want you can remove the cloned project directory and the `\u003Cpath to a location where to store ssh key-pairs on your PC>` directory.\n\nIf you would like to deploy the demo once again without removing the directory with the cloned repo dont forget to remove files created during the previous demo deployment, namely `terraform.tfstate` files in the root directory and `.git` directories everywhere in the `repo_content` directory.\n\nIn the [second part](/blog/how-to-migrate-atlassians-bamboo-servers-ci-cd-infrastructure-to-gitlab-ci-part-two/) of this tutorial, we'll look at a real-world example of how this can work.\n\n\n\n\n\n",[108,789,9],{"slug":3398,"featured":6,"template":700},"migration-from-atlassian-bamboo-server-to-gitlab-ci","content:en-us:blog:migration-from-atlassian-bamboo-server-to-gitlab-ci.yml","Migration From Atlassian Bamboo Server To Gitlab Ci","en-us/blog/migration-from-atlassian-bamboo-server-to-gitlab-ci.yml","en-us/blog/migration-from-atlassian-bamboo-server-to-gitlab-ci",{"_path":3404,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3405,"content":3411,"config":3417,"_id":3419,"_type":13,"title":3420,"_source":15,"_file":3421,"_stem":3422,"_extension":18},"/en-us/blog/migration-guide-github-advanced-security-to-gitlab-ultimate",{"title":3406,"description":3407,"ogTitle":3406,"ogDescription":3407,"noIndex":6,"ogImage":3408,"ogUrl":3409,"ogSiteName":685,"ogType":686,"canonicalUrls":3409,"schema":3410},"Migration guide: GitHub Advanced Security to GitLab Ultimate","Understand the similarities and differences between GitLab Ultimate and GitHub Advanced Security. Then follow this in-depth tutorial to make the move to the GitLab DevSecOps platform.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666187/Blog/Hero%20Images/blog-image-template-1800x945__6_.png","https://about.gitlab.com/blog/migration-guide-github-advanced-security-to-gitlab-ultimate","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Migration guide: GitHub Advanced Security to GitLab Ultimate\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2024-05-01\",\n      }",{"title":3406,"description":3407,"authors":3412,"heroImage":3408,"date":3413,"body":3414,"category":830,"tags":3415},[1444],"2024-05-01","GitLab is the most comprehensive AI-powered DevSecOps platform, enabling organizations to deliver more secure software faster with one platform for your entire software delivery lifecycle. GitHub provides an Advanced Security add-on, which enables additional security features within GitHub. However, it lacks the depth and breadth of security features provided natively by GitLab. Organizations looking to migrate to GitLab Ultimate to enhance their security across all areas of the SDLC can use this guide to compare the two offerings and as a tutorial to move to the GitLab platform.\n\nThis article includes:\n\n- [A comparison between GitLab Ultimate and GitHub Advanced Security](#a-comparison-between-gitlab-ultimate-and-github-advanced-security)\n- [How to migrate a GitHub repository to GitLab](#how-to-migrate-a-github-repository-to-gitlab)\n- [How to migrate from GitHub Advanced Security to GitLab Ultimate feature-by-feature](#how-to-migrate-feature-by-feature)\n- [An introduction to additional GitLab Ultimate's security features](#additional-gitlab-ultimate-security-features)\n\n## A comparison between GitLab Ultimate and GitHub Advanced Security\n\n[GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/) is GitLab's top subscription tier for enterprises looking to deliver secure software faster. GitHub Advanced Security is an add-on to GitHub Enterprise, which enables additional security features.\n\n### Similarities between GitLab Ultimate and GitHub Advanced Security\n\nGitLab Ultimate and GitHub Advanced Security both provide:\n- Static Application Security Testing ([SAST](https://docs.gitlab.com/ee/user/application_security/sast/)), secret scanning, and dependency scanning\n- contextual vulnerability intelligence and resolution advice\n- a list of dependencies or software bill of materials ([SBOM](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/))\n- security metrics and insights\n\n### Differences between GitLab Ultimate and GitHub Advanced Security\n\nGitLab Ultimate differs from GitHub Advanced Security in the following ways:\n\n- GitLab natively provides additional code scanners such as container scanning, Dynamic Application Security Testing ([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)), Web API fuzz testing, and more. These scanners are a mix of optimized proprietary and open source technologies with custom rulesets. For a full list, see the [GitLab AppSec documentation](https://docs.gitlab.com/ee/user/application_security/secure_your_application.html).\n- GitLab provides [granular security guardrails](https://docs.gitlab.com/ee/user/application_security/policies/) to prevent insecure code from being merged without approval.\n- GitLab security scanners can be run in [air-gapped or limited-connectivity environments](https://docs.gitlab.com/ee/user/application_security/offline_deployments/).\n- GitLab provides the [Compliance Center](https://docs.gitlab.com/ee/user/compliance/compliance_center/), which enables oversight of compliance violations across an entire organization.\n\nGitLab Ultimate also provides additional security and compliance capabilities, portfolio and value stream management, live upgrade assistance, and more. See the [GitLab Ultimate documentation](https://about.gitlab.com/pricing/ultimate/) to learn more about these additional features.\n\n## How to migrate a GitHub repository to GitLab\n\nGitLab provides a built-in importer, which allows you to import your GitHub projects from either GitHub.com or GitHub Enterprise to GitLab. The importer allows you to migrate not only the GitHub Repository to GitLab, but several other objects, including issues, collaborators (members), and pull requests. For a complete list of what can be migrated, see the [GitHub imported data documentation](https://docs.gitlab.com/ee/user/project/import/github.html#imported-data). You can perform the migration as follows:\n1. On the left sidebar, at the top, select **Create new (+)**.\n2. Select **New project/repository** under the **In GitLab** section.\n3. Select **Import project**.\n\n![Import project selection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/1-Import-Project.png)\n\n4. Press the **GitHub** button.\n    - If using GitLab self-managed, then you must [enable the GitHub importer](https://docs.gitlab.com/ee/administration/settings/import_and_export_settings.html#configure-allowed-import-sources).\n    - Note that other importers can be initiated in the same way.\n5. Now, you can do one of the following:\n    - Authorize with GitHub Oauth by selecting **Authorize with GitHub**.\n    - Use a GitHub personal access token:\n       - Go to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new).\n       - In the **Note** field, enter a token description.\n       - Select the **repo** scope.\n       - Optionally, to import Collaborators, select the **read:org** scope.\n       - Press the **Generate token** button.\n       - On the GitLab import page, in the Personal Access Token field, paste the GitHub personal access token.\n6. Press the **Authenticate** button.\n7. Select the items you wish to migrate.\n8. Select the projects you wish to migrate and to where.\n9. Press the **Import** button.\n\nYour imported project should now be in your workspace. For additional guidance on migrating from GitHub to GitLab, watch this video:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/0Id5oMl1Kqs?si=HEpZVy94cpfPfAky\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nYou can also perform the migration using a [GitHub personal access token](https://docs.gitlab.com/ee/user/project/import/github.html#use-a-github-personal-access-token) or the [GitLab REST API](https://docs.gitlab.com/ee/user/project/import/github.html#use-the-api). The importer also allows importing from other sources such as Bitbucket or Gitea. To learn more, read the [importer documentation](https://docs.gitlab.com/ee/user/project/import/).\n\n## How to migrate feature-by-feature\n\nLet’s go over how to leverage each feature provided by GitHub Advanced Security in GitLab Ultimate. You must have a [GitLab Ultimate license](https://about.gitlab.com/pricing/ultimate/) to continue. GitLab provides a [free trial](https://about.gitlab.com/free-trial/devsecops/) to get you started.\n\n### Code scanning\nGitHub provides code scanning to provide contextual vulnerability intelligence and advice for static source code. The same can be done within GitLab by enabling [SAST](https://docs.gitlab.com/ee/user/application_security/sast/). GitLab SAST scanners cover a wider set of programming languages and frameworks than GitHub’s [CodeQL](https://docs.github.com/en/code-security/code-scanning/introduction-to-code-scanning/about-code-scanning-with-codeql#about-codeql).\n\nTo enable code scanning in GitLab, you can simply add the [SAST template](https://docs.gitlab.com/ee/user/application_security/sast/#configure-sast-in-your-cicd-yaml) to your `.gitlab-ci.yml`:\n\n```yaml\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n```\n\nOnce the template has been added, any time new code is checked in, SAST will auto-detect the [programming languages](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks ) used in your project. It will then scan the source code for known vulnerabilities.\n\n**Note:** Security scanners can also be added to your project using GitLab's [security configuration](https://docs.gitlab.com/ee/user/application_security/configuration/), which can automatically create a merge request to update your pipeline. To learn more, see the [Configure SAST by using the UI documentation](https://docs.gitlab.com/ee/user/application_security/sast/#configure-sast-by-using-the-ui).\n\nSAST results of the diff between the feature-branch and the target-branch display in the merge request widget. The merge request widget displays SAST results and resolutions that were introduced by the changes made in the merge request.\n\n![Security scanning in merge request](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/2-SAST-MR-View.png)\n\nEach vulnerability displays data to assist with remediation, including detailed description, severity, location, and resolution information:\n\n![SAST vulnerability details](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/3-SAST-MR-View-Detailed.png)\n\nYou can take action on these vulnerabilities:\n\n- **Dismiss vulnerability**: Allows a developer to dismiss the vulnerability with a comment. This assists the security team performing a review.\n- **Create issue**: Allows an issue to be created to keep track of a vulnerability that requires additional oversight.\n\nThese changes can also be seen inline when changing to the **Changes** view within the merge request.\n\n![SAST vulnerability changes view](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/4-SAST-MR-View-Changes.png)\n\n#### Customizing SAST scanners\n\nGitLab allows you to override a SAST job definition so you can change properties like variables, dependencies, or rules. You can do this by declaring a job with the same name as the SAST job to override. Then, place this new job after the template inclusion and specify any additional keys under it.\n\nFor example, the following configuration:\n- overwrites the version the `semgrep-sast` scanner uses\n- runs a script to fetch modules from private projects before running `gosec-sast`\n- configures all scanners to search at a maximum depth of 10\n\n```yaml\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n\nvariables:\n  SEARCH_MAX_DEPTH: 10\n\nsemgrep-sast:\n  variables:\n    SAST_ANALYZER_IMAGE_TAG: \"3.7\"\n\ngosec-sast:\n  before_script:\n    - |\n      cat \u003C\u003CEOF > ~/.netrc\n      machine gitlab.com\n      login $CI_DEPLOY_USER\n      password $CI_DEPLOY_PASSWORD\n      EOF\n```\n\n**Note:** The available SAST jobs can be found in the [`SAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml). Configurations can be found in the [Available SAST CI/CD variables documentation](https://docs.gitlab.com/ee/user/application_security/sast/#available-cicd-variables).\n\n#### Customizing SAST rulesets\n\nFor each SAST analyzer, GitLab processes the code then uses rules to find possible weaknesses in source code. These rules determine what types of weaknesses the scanner reports.\n\n- For Semgrep-based SAST scanners, GitLab creates, maintains, and supports the rules that are used. It combines the Semgrep open source engine, GitLab-managed detection rules, and GitLab proprietary technology for vulnerability tracking and false positive detection.\n- For other SAST analyzers, the rules are defined in the upstream projects for each scanner.\n\nYou can customize the behavior of the SAST scanners by defining a ruleset configuration file in the repository being scanned:\n- Disable predefined rules (available for all analyzers)\n- Override predefined rules (available for all analyzers)\n- Replace predefined rules by synthesizing a custom configuration using passthroughs\n\nFor more information and examples on configuring SAST rules, see the [SAST rules](https://docs.gitlab.com/ee/user/application_security/sast/rules.html) and [Customizing rulesets documentation](https://docs.gitlab.com/ee/user/application_security/sast/customize_rulesets.html).\n\n### Secret scanning\n\nGitHub provides secret scanning, which can find, block, and revoke leaked secrets. The same can be done within GitLab by enabling [Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/).\n\nTo enable Secret Detection in GitLab, you can simply add the following template to your `.gitlab-ci.yml`:\n\n```yaml\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n```\n\nOnce the template has been added, any time new code is checked in (or a pipeline is run), the secret scanner will scan the source code for known secrets. Pipeline Secret Detection scans different aspects of your code, depending on the situation. For all methods except the “Default branch”, Pipeline Secret Detection scans commits, not the working tree. See the [Secret detection coverage documentation](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/#coverage) to learn more about how secret scanning works.\n\nWhen creating a merge request, Secret Detection scans every commit made on the source branch. Just like in SAST, each detected vulnerability provides the following information (such as location) and identifiers to assist with the remediation process:\n\n![Secret Detection vulnerability details](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/5-Secret-Detection-MR-Detailed.png)\n\nSimilar to SAST, you can take action on these vulnerabilities straight from the merge request, including dismissing vulnerabilities, and creating issues.\n\n#### Customizing Secret Detection jobs\n\nGitLab allows you to override a Secret Detection job definition so you change properties like variables, dependencies, or rules. You can do this by declaring a job with the same name as the Secret Detection job. Then place this new job after the template inclusion and specify any additional keys under it. For example, the following configuration:\n\n- overwrites the stage the secret detection job runs on to `security`\n- enables the historic scanning\n- changes the Secrets Analyzer version to 4.5\n\n```yaml\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n\nsecret_detection:\n  stage: security\n  variables:\n    SECRET_DETECTION_HISTORIC_SCAN: \"true\"\n    SECRETS_ANALYZER_VERSION: \"4.5\"\n```\n\n**Note:** The available Secret Detection jobs can be found in the [SAST.gitlab-ci.yml template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml). Available configurations can be found in the [Available Secret Detection CI/CD variables documentation](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/#customizing-analyzer-settings).\n\n#### Customizing Secret Detection rulesets\n\nThe Secret Detection analyzer allows you to customize which secrets are reported in the GitLab UI. The following customization options can be used separately, or in combination:\n\n- disable predefined rules\n- override predefined rules\n- synthesize a custom configuration\n- specify a remote configuration file\n\nFor example, by creating the file `.gitlab/secret-detection-ruleset.toml`, in the root directory of your project, the default GitLeaks package is extended to ignore test tokens from detection:\n\n```yaml\n### extended-gitleaks-config.toml\ntitle = \"extension of gitlab's default gitleaks config\"\n\n[extend]\n### Extends default packaged path\npath = \"/gitleaks.toml\"\n\n[allowlist]\n  description = \"allow list of test tokens to ignore in detection\"\n  regexTarget = \"match\"\n  regexes = [\n    '''glpat-1234567890abcdefghij''',\n  ]\n```\n\nFor more information on overriding the predefined analyzer rules, check out the [Secret Detection documentation](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/#override-predefined-analyzer-rules).\n\n#### Automatic response to leaked secrets\n\nGitLab Secret Detection automatically responds when it finds certain types of leaked secrets. Automatic responses can:\n- automatically revoke the secret\n- notify the partner that issued the secret and the partner can then revoke the secret, notify its owner, or otherwise protect against abuse\n\nGitLab can also notify partners when credentials they issue are leaked in public repositories on GitLab.com. If you operate a cloud or SaaS product and you’re interested in receiving these notifications, you can implement a Partner API, which is called by the GitLab Token Revocation API.\n\nSee the [Automatic response to leaked secrets documentation](https://docs.gitlab.com/ee/user/application_security/secret_detection/automatic_response.html) to learn more.\n\n### Supply chain security\n\nGitHub enables you to secure, manage, and report on software supply chains with automated security and version updates and one-click SBOMs. GitLab can meet your supply chain security needs using the Dependency Scanning and Dependency List (SBOM) features.\n\nTo enable Dependency Scanning in GitLab, you can simply add the following template to your `.gitlab-ci.yml`:\n\n```yaml\ninclude:\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml\n```\n\nOnce the template has been added, any time new code is checked in, Dependency Scanning will auto-detect the [package managers](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers) used in your project. It will then scan the dependencies used for known vulnerabilities.\n\nDependency Scanning results of the diff between the feature-branch and the target-branch display in the merge request widget. The merge request widget displays Dependency Scanning results and resolutions that were introduced by the changes made in the merge request. Within a merge request, each vulnerability displays relevant information to assist with remediation such as identifiers, evidence, and solutions:\n\n![Dependency Scanner vulnerability details](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/6-Dependency-Scanner-MR-View-Detailed.png)\n\nSimilar to SAST and Secret Detection, you can take action on these vulnerabilities straight from the merge request, including dismissing vulnerabilities and creating issues.\n\n#### Configuring Dependency Scanning\n\nTo override a job definition (for example, to change properties like variables or dependencies), declare a new job with the same name as the one to override. Place this new job after the template inclusion and specify any additional keys under it. For example, the following code:\n\n- disables automatic remediation of vulnerable dependencies\n- requires a build job to complete before Dependency Scanning\n\n```yaml\ninclude:\n  - template: Jobs/Dependency-Scanning.gitlab-ci.yml\n\ngemnasium-dependency_scanning:\n  variables:\n    DS_REMEDIATE: \"false\"\n  dependencies: [\"build\"]\n```\n\nTo learn more about configuring the dependency scanners, see the [Customizing analyzer behavior documentation](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-analyzer-behavior).\n\n#### Generating an SBOM\n\nGitLab provides a Dependency List (SBOM) to review your project or group dependencies and key details about those dependencies, including their known vulnerabilities. This list is a collection of dependencies in your project, including existing and new findings. The Dependency List is generated after the dependency scanner runs successfully on the [default branch](https://docs.gitlab.com/ee/user/project/repository/branches/default.html). To access the Dependency List:\n\n1. On the left sidebar, select **Search or go to** and find your project.\n2. Select **Secure > Dependency List**.\n\n![Dependency list (SBOM)](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/7-Dependency-List.png)\n\nFrom here you can see the following information on your dependencies:\n\n| Field\t| Description |\n| ----- | ----------- |\n| Component\t| The dependency’s name and version. |\n| Packager | The packager used to install the dependency. |\n| Location | For system dependencies, this lists the image that was scanned. For application dependencies, this shows a link to the packager-specific lock file in your project that declared the dependency. It also shows the dependency path to a top-level dependency, if any, and if supported. |\n| License | Links to dependency’s software licenses. A warning badge that includes the number of vulnerabilities detected in the dependency. |\n| Projects | Links to the project with the dependency. If multiple projects have the same dependency, the total number of these projects is shown. To go to a project with this dependency, select the Project's number, then search for and select its name. The project search feature is supported only on groups that have up to 600 occurrences in their group hierarchy. |\n\n\u003Cp>\u003C/p>\n\nSee the [Dependency List documentation](https://docs.gitlab.com/ee/user/application_security/dependency_list/) to learn more.\n\n### Security and compliance administration\n\nGitHub Advanced Security allows you to view security metrics and insights and assess code security risk. Now let’s examine how to do the same with GitLab Ultimate.\n\n#### Viewing security metrics and insights\n\nGitLab provides [Security dashboards](https://docs.gitlab.com/ee/user/application_security/security_dashboard/) to help assess the security posture of your applications. These dashboards display a collection of metrics, ratings, and charts for the vulnerabilities detected by the security scanners run on your project:\n\n- vulnerability trends over a 30-, 60-, or 90-day timeframe for all projects in a group\n- a letter grade rating for each project based on vulnerability severity\n- the total number of vulnerabilities detected within the past 365 days, including their severity\n\nTo access the Security dashboard:\n\n1. On the left sidebar, select **Search or go to** and find your project or group.\n2. From the side tab, select **Secure > Security** dashboard.\n3. Filter and search for what you need.\n\nThe group view displays your security posture for all projects in your group:\n\n![Group Security dashboard](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/8-SD-Group.png)\n\nThe project view displays your security posture for just the project:\n\n![Project Security dashboard](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/9-SD-Project.png)\n\n#### Assess code security risk\n\nGitLab Ultimate features a [Vulnerability Report](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/), which provides information about vulnerabilities from scans of the default branch. It contains cumulative results of all successful jobs, regardless of whether the pipeline was successful. At all levels, the Vulnerability Report contains:\n\n- totals of vulnerabilities per severity level\n- filters for common vulnerability attributes\n- details of each vulnerability, presented in tabular layout\n\n![Vulnerability Report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/10-Vulnerability-Report.png)\n\nClicking on a vulnerability enables access to its [Vulnerability Page](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/), which contains details of the vulnerability including a description, location, identifiers, and more. Below is an example of the Vulnerability Page for an SQL Injection vulnerability detected by our SAST scanner:\n\n![SQL Injection Vulnerability Page](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/11-Vulnerability-Page-1.png)\n\nFrom here the security team can collaborate by [changing the status of a vulnerability](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#change-the-status-of-a-vulnerability) along with a reason and [creating issues to better track changes](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#create-a-gitlab-issue-for-a-vulnerability).\n\nFrom the Vulnerability Page, you can also leverage [GitLab Duo](https://about.gitlab.com/gitlab-duo/), our AI-powered suite of features, to explain the vulnerability and [automatically create a merge request that resolves the vulnerability](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#vulnerability-resolution).\nGitLab Duo's [Vulnerability Explanation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#vulnerability-explanation) uses a large language model to:\n\n- summarize the vulnerability.\n- help developers and security analysts to understand the vulnerability, how it could be exploited, and how to fix it\n- provide a suggested mitigation\n\n![SQL Injection GitLab Duo AI explanation](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/13-Explain-Vulnerability.png)\n\n## Additional GitLab Ultimate security features\n\nGitLab Ultimate contains many more security features that cannot be found within GitHub Advanced Security. A few examples of these additional security features are: additional security scanners for the complete software development lifecycle (SDLC), granular security guardrails, and custom permissions.\n\n### Security scanners for the entire SDLC\n\nOur portfolio of security scanners extends spans the SDLC.\n\n| Scanner Name | Scans | Languages/Files scanned |\n|  -------------- | ----- | ------------------------- |\n| [Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/) | Static source code | C/C++, Java, Python, Go, JavaScript, C#, and more |\n| [Dynamic Application Security Testing (DAST)](https://docs.gitlab.com/ee/user/application_security/dast/) | Running web application, live API | Language-agnostic |\n| [Infrastructure as Code (IaC) Scanning](https://docs.gitlab.com/ee/user/application_security/iac_scanning/) | IaC files |Terraform, AWS Cloud Formation, Ansible, and more |\n| [Container Scanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/) | Static and running container images | Dockerfile |\n| [Dependency Scanning and License Scanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/) | Application dependencies | Requirements.txt, Yarn, Gradle, Npm, and more |\n| [Web API Fuzz Testing](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/) | Sends random/malformed data to web-api | OpenAPI, GraphQL, HAR, Postman Collection |\n| [Coverage-guided Fuzz Testing](https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/) | Sends random/malformed data to function | C/C++, Go, Swift, Python, Rust, Java, JavaScript, AFL |\n\n\u003Cp>\u003C/p>\n\nGitLab also allows you to integrate [third-party scanners](https://about.gitlab.com/blog/integrate-external-security-scanners-into-your-devsecops-workflow/) and [custom scanners](https://about.gitlab.com/blog/how-to-integrate-custom-security-scanners-into-gitlab/) into the platform. Once integrated, the scanner results are automatically presented in various places in GitLab, such as the Pipeline view, merge request widget, and Security dashboard. See the [Security Scanner Integration documentation](https://docs.gitlab.com/ee/development/integrations/secure.html) to learn more.\n\n### Granular security and compliance policies\n\nPolicies in GitLab provide security and compliance teams with [a way to enforce controls globally in their organization](https://about.gitlab.com/blog/meet-regulatory-standards-with-gitlab/). Security teams can ensure:\n\n- security scanners are enforced in development team pipelines with proper configuration\n- all scan jobs execute without any changes or alterations\n- proper approvals are provided on merge requests based on results from those findings\n\n![Merge Request Security Policies](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/14-MR-Policy.png)\n\nCompliance teams can centrally enforce multiple approvers on all merge requests and ensure various settings are enabled on projects in scope of organizational requirements, such as enabling or locking merge request and repository settings. To learn more see the [GitLab Security Policy](https://docs.gitlab.com/ee/user/application_security/policies/) documentation.\n\n### Custom roles and granular permissions\n\n[GitLab Ultimate provides custom roles](https://about.gitlab.com/blog/how-to-tailor-gitlab-access-with-custom-roles/), which allow an organization to create user roles with the precise privileges and permissions required for that organization’s needs.\n\nFor example, a user could create a “Security Auditor” role with permissions to view security vulnerabilities in the system, but not be able to view source code, nor perform any changes within the repository. This granular set of permissions enables well-defined separation of duties.\n\n![Custom role creation](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/15-Custom-Roles.png)\n\nTo learn more see the [Custom Roles](https://docs.gitlab.com/ee/user/custom_roles.html) and [available Granular Permissions](https://docs.gitlab.com/ee/user/custom_roles/abilities.html) documentation.\n\n### Compliance Center\n\nThe Compliance Center is the central location for compliance teams to manage their compliance standards’ adherence reporting, violations reporting, and compliance frameworks for their group. The Compliance Center includes the following:\n\n- [Compliance standards adherence dashboard](https://docs.gitlab.com/ee/user/compliance/compliance_center/compliance_standards_adherence_dashboard.html) lists the adherence status of projects complying to the GitLab standard.\n- [Compliance violations report](https://docs.gitlab.com/ee/user/compliance/compliance_center/compliance_violations_report.html) shows a high-level view of merge request activity for all projects in the group.\n- [Compliance frameworks report](https://docs.gitlab.com/ee/user/compliance/compliance_center/compliance_frameworks_report.html) shows all the compliance frameworks in a group.\n- [Compliance projects report](https://docs.gitlab.com/ee/user/compliance/compliance_center/compliance_projects_report.html) shows the compliance frameworks that are applied to projects in a group.\n\n![Compliance Center](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674404/Blog/Content%20Images/16-Compliance-Center.png)\n\nThese dashboards assist with making sure separation of duties is being followed to optimize compliance within your organization. To learn more see the [Compliance Center documentation](https://docs.gitlab.com/ee/user/compliance/compliance_center/).\n\n## Read more\n\nThis article covers only a portion of the wide range of security features GitLab Ultimate offers. Check out these resources to learn more about how GitLab Ultimate can help enhance your organizational security and developer efficiency:\n\n- [Why GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/)\n- [Getting Started with DevSecOps Tutorial](https://gitlab-da.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/)\n- [Getting Started with DevSecOps Sample Project](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\n- [Import your project from GitHub to GitLab documentation](https://docs.gitlab.com/ee/user/project/import/github.html)\n- [Migrating from GitHub Actions documentation](https://docs.gitlab.com/ee/ci/migration/github_actions.html)\n- [Tutorial: Create and run your first GitLab CI/CD pipeline](https://docs.gitlab.com/ee/ci/quick_start/)\n- [Tutorial: Create a complex pipeline](https://docs.gitlab.com/ee/ci/quick_start/tutorial.html)\n- [CI/CD YAML syntax reference](https://docs.gitlab.com/ee/ci/yaml/)",[9,3416,830,495,1057],"zero trust",{"slug":3418,"featured":90,"template":700},"migration-guide-github-advanced-security-to-gitlab-ultimate","content:en-us:blog:migration-guide-github-advanced-security-to-gitlab-ultimate.yml","Migration Guide Github Advanced Security To Gitlab Ultimate","en-us/blog/migration-guide-github-advanced-security-to-gitlab-ultimate.yml","en-us/blog/migration-guide-github-advanced-security-to-gitlab-ultimate",{"_path":3424,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3425,"content":3431,"config":3436,"_id":3438,"_type":13,"title":3439,"_source":15,"_file":3440,"_stem":3441,"_extension":18},"/en-us/blog/modernizing-a-simple-c-application-to-java-with-gitlab-duo",{"title":3426,"description":3427,"ogTitle":3426,"ogDescription":3427,"noIndex":6,"ogImage":3428,"ogUrl":3429,"ogSiteName":685,"ogType":686,"canonicalUrls":3429,"schema":3430},"Modernizing a simple C++ application to Java with GitLab Duo","Learn how to refactor code from memory unsafe languages to memory safe languages with the help of GitLab's AI capabilities, saving time and effort on application modernization.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659507/Blog/Hero%20Images/AdobeStock_623844718.jpg","https://about.gitlab.com/blog/modernizing-a-simple-c-application-to-java-with-gitlab-duo","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Modernizing a simple C++ application to Java with GitLab Duo\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2024-12-11\",\n      }",{"title":3426,"description":3427,"authors":3432,"heroImage":3428,"date":3433,"body":3434,"category":718,"tags":3435},[715],"2024-12-11","Memory unsafe languages are those that do not handle any memory management\non behalf of the developer. For example, when programming in C or C++, if\nyou need memory during runtime, you will need to allocate and deallocate the\nmemory yourself, running the risk of ending up with memory leaks in cases\nwhen you inadvertently forget to deallocate it. Other languages like Ada and\nFORTRAN provide some memory management but may not prevent memory leaks.\nMany organizations, including those in the public sector, have applications\nthat have been developed using languages that are memory unsafe and are\noften looking to modernize these to a memory safe language, such as Java,\nPython, JavaScript, or Golang.\n\n\nThis tutorial focuses on a specific example of modernizing a simple C++\napplication to Java by refactoring it with the help of [GitLab\nDuo](https://about.gitlab.com/gitlab-duo/), our suite of AI capabilities,\nand shows how much time and effort you can save in the migration.\n\n\n## Understanding the simple C++ application\n\n\nLet’s make the assumption that we have been tasked with the migration of a\nC++ application to a memory safe language, namely Java. The C++ application\ncan be found in the following project (thank you to\n[@sugaroverflow](https://gitlab.com/sugaroverflow) for contributing this\nsample application):\n\n\n[https://gitlab.com/gitlab-da/use-cases/ai/ai-applications/refactor-to-java/air-quality-application](https://gitlab.com/gitlab-da/use-cases/ai/ai-applications/refactor-to-java/air-quality-application)\n\n\nSince this is the first time we are seeing this application, let’s invoke\nGitLab Duo Code explanation to better understand what it does. We open file\n`main.cpp` in Visual Studio Code and select the entirety of this file. We\nthen right-click and select **GitLab Duo Chat > Explain selected snippet**\nfrom the popup menu.\n\n\n![duo-code-explanation-menu-option](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675546/Blog/Content%20Images/code-explanation-menu-option.png)\n\n\nThe GitLab Duo Chat window opens up and the slash command `/explain` is\nexecuted for the selected code. Chat returns a very thorough and detailed\ndescription and explanation in natural language form of what each function\ndoes in the file as well as examples on how to run the compiled program.\n\n\n![code-explanation-text](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/code-explanation-text.png)\n\n\nIn short, the simple C++ application takes a U.S. zip code as input and\nreturns the air quality index for that zip code.\n\n\n## Compiling and running the C++ application\n\n\nTo further understand this simple C++ application, we proceed to compile and\nrun it. We could have asked Chat how to do this, however, the project has a\nREADME file that provides the commands to compile the project, so we go\nahead and use those by entering them in the Terminal window of VS Code.\n\n\n![compile-command](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/compile-command.png)\n\n\nAfter the compilation finishes, we change directory to the `build`\nsubdirectory in the project, which is where the compilation process places\nthe executable file for this application. Then, we run the executable by\nentering the following command:\n\n\n`./air_quality_app 32836`\n\n\nAnd we see the response as follows:\n\n\n`Air Quality Index (AQI) for Zip Code 32836: 2 (Fair)`\n\n\n![cplus-plus-app-execution-output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/cplus-plus-app-execution-output.png)\n\n\nThis confirms to us that the application was successfully compiled and it’s\nexecuting appropriately.\n\n\n## Refactoring the application to Java\n\n\nLet’s start migrating this C++ application to Java. We take advantage of\nGitLab Duo Chat and its refactoring capabilities by using the slack command\n`/refactor`. We qualify the slash command with specific instructions on what\nto do for the refactoring. We enter the following command in the Chat input\nfield:\n\n\n> /refactor this entire application to Java. Provide its associated pom.xml\nto build and run the Java application. Also, provide the directory structure\nshowing where all the resulting files should reside for the Java\napplication.\n\n\n![refactor-chat-output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/refactor-chat-output.png)\n\n\nChat returns a set of Java files that basically refactor the entire C++\napplication to the memory safe language. In addition and per the prompt,\nChat returns the pom.xml file, needed by\n[maven](https://docs.gitlab.com/ee/api/packages/maven.html) for the building\nand execution of the refactored application as well as its directory\nstructure, indicating where each generated file should reside.\n\n\nWe copy and save all the generated files to our local directory.\n\n\n## Creating the Java project\n\n\nIn VS Code, we now proceed to open an empty project in which we will set up\nthe directory structure of the new Java application and its contents.\n\n\nWe create all the previously generated Java files in their corresponding\ndirectories in the new project and paste their contents in each.\n\n\n![java-files-created](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/java-files-created.png)\n\n\nLastly, we save all the files to our local disk.\n\n\n## Asking for help to build and run the Java application\n\n\nAt this point, we have an entire Java application that has been refactored\nfrom C++. Now, we need to build it but we don’t quite remember what maven\ncommand we need to use to accomplish this.\n\n\nSo we ask GitLab Duo Chat about this. We enter the following prompt in the\nChat input field:\n\n\n> How do you build and run this application using maven?\n\n\n![maven-info-output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/maven-info-output.png)\n\n\nChat returns with a thorough explanation on how to do this, including\nexamples of the maven command to build and run the newly created Java\napplication.\n\n\n## Building and running the Java application\n\n\nGitLab Duo Chat understands the application and environment context and\nresponds that we first need to create an environment variable called\n`API_KEY` before we can run the application.\n\n\nIt also provides the maven command to execute to build the application,\nwhich we enter in the Terminal window:\n\n\n```unset\n\nmvn clean package\n\n``` \n\n\n![java-build-output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/java-build-output.png)\n\n\nOnce the build finishes successfully, we copy the generated command to run\nthe application from the Chat window and paste it in the Terminal window:\n\n\n```unset\n\njava -jar target/air-quality-checker-1.0-SNAPSHOT-jar-with-dependencies.jar\n90210\n\n```\n\n\n![java-app-execution-output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675547/Blog/Content%20Images/java-app-execution-output.png)\n\n\nThe application successfully executes and returns the string:\n\n\n```unset\n\nAir Quality Index (AQI) for Zip Code 90210: 2 (Fair)\n\n```\n\n\nWe have confirmed that the modernized version of the application, now\nrefactored in Java, runs just like its original C++ version.\n\n\n## Watch this tutorial in action\n\n\nWe have seen that by leveraging the power of GitLab Duo in your\nmodernization activities, you can save a great deal of time and effort,\nfreeing you to spend more time innovating and creating value to your\norganization.\n\n\nHere is a video to show you, in action, the tutorial you just read:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/LJ7GOr_P0xs?si=_ZjF75DAXEQnY2Mn\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n> #### Want to get started with GitLab Duo? [Start a free trial\ntoday!](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/)\n\n\n## Learn more\n\n- [Refactor code into modern languages with AI-powered GitLab\nDuo](https://about.gitlab.com/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo/)\n\n- [Secure by Design principles meet DevSecOps innovation in GitLab\n17](https://about.gitlab.com/blog/secure-by-design-principles-meet-devsecops-innovation-in-gitlab-17/)\n\n- [How to secure memory-safe vs. manually managed\nlanguages](https://about.gitlab.com/blog/memory-safe-vs-unsafe/)\n",[720,495,9,185],{"slug":3437,"featured":6,"template":700},"modernizing-a-simple-c-application-to-java-with-gitlab-duo","content:en-us:blog:modernizing-a-simple-c-application-to-java-with-gitlab-duo.yml","Modernizing A Simple C Application To Java With Gitlab Duo","en-us/blog/modernizing-a-simple-c-application-to-java-with-gitlab-duo.yml","en-us/blog/modernizing-a-simple-c-application-to-java-with-gitlab-duo",{"_path":3443,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3444,"content":3450,"config":3456,"_id":3458,"_type":13,"title":3459,"_source":15,"_file":3460,"_stem":3461,"_extension":18},"/en-us/blog/mr-reviews-with-vs-code",{"title":3445,"description":3446,"ogTitle":3445,"ogDescription":3446,"noIndex":6,"ogImage":3447,"ogUrl":3448,"ogSiteName":685,"ogType":686,"canonicalUrls":3448,"schema":3449},"How to do GitLab merge request reviews in VS Code","Code review is critical to modern software development. We're making it easier by bringing merge request reviews right into VS Code.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666775/Blog/Hero%20Images/cover.jpg","https://about.gitlab.com/blog/mr-reviews-with-vs-code","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to do GitLab merge request reviews in VS Code\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tomas Vik\"}],\n        \"datePublished\": \"2021-01-25\",\n      }",{"title":3445,"description":3446,"authors":3451,"heroImage":3447,"date":3453,"body":3454,"category":785,"tags":3455},[3452],"Tomas Vik","2021-01-25","\n\nThis post will give you an idea of how VS Code can aid your code review process. You'll get an overview of the features that GitLab VS Code Extension currently supports, as well as what we plan to introduce in the future.\n\nReviewing merge requests is a core part of GitLab: both the product (since [version 2.0.0](https://gitlab.com/gitlab-org/gitlab/blob/6a3621202e3f7274150862198f59d2579c326650/changelogs/archive.md#L7222), released in 2011) and the company. We recognize that certain review tasks are hard to do just by looking at the diff, and we strive to make them easier. One such task might be looking in the codebase for duplicated code or examples of a particular coding style.\n\nWe decided to aid code reviewers in two ways:\n\n## First way: The GitLab Web IDE\n\nFirst, we introduced the [Web IDE](/blog/introducing-gitlab-s-integrated-development-environment/), which helps our users work [with the codebase in the browser](/direction/create/ide/web_ide/#overview). You can quickly open multiple files, make changes, and commit them. The Web IDE is handy when you need to make a small change, or you don't have the project cloned locally.\n\nThe second way is more recent. We always wanted to bring the code review experience closer to code editors, where developers spend a large portion of their time. But the editor market is very fragmented (you find out the hard way if Emacs and Vim users meet at a party). And it isn't feasible to build GitLab support into all major editors (however, there are plenty of editor plugins maintained by the community[^1]). \n\n## Second way: Bringing code reviews into the editor\n\nRecently, as [VS Code gained a significant user share](https://insights.stackoverflow.com/survey/2019#development-environments-and-tools), it started to make sense to [commit to maintaining the GitLab VS Code extension](/blog/use-gitlab-with-vscode/), which was started as a community project by one, at the time, GitLab employee: [Fatih](https://gitlab.com/fatihacet). After an initial housekeeping period, we started chipping away tasks that will ultimately bring the code review experience into the editor.\n\nIn my previous post I talked about the great [VS Code Extension API](/blog/vscode-extension-development-with-gitlab/). This API gives extensions almost full control over the editor. When the API introduced commenting functionality two years ago, extensions could start contributing comments to the editor windows. These comments are shown similarly as comments on a Google Doc. Being able to natively show comments is perfect for reviewing code changes in the editor and other extensions that provide code reviews are already using this commenting API[^2].\n\n![Merge request review in VS Code](https://about.gitlab.com/images/blogimages/mr-reviews-with-vs-code/full-mr-review-screen.png){: .shadow.medium.center}\nMerge request review in VS Code\n{: .note .text-center}\n\nOver the last few milestones, we started showing MR changes in VS Code and even showing discussions on these. This means that you can open an MR in your editor and read through the code and comments without switching windows and context. I find this really useful because I can still interact with my editor the way I'm used to, even as I'm reviewing MRs. I can use full-text search to find if the MR duplicates existing code or I can open a different test file and compare whether the code style matches.\n\nCurrently, the interaction with MR is mostly read-only. That means you can see the changes and discussions, but you can't add or change comments, yet[^3]. But even in this current form, you can benefit from having the VS Code functionality so close to your review, especially for the initial understanding of the change.\n\n![VS Code supports Markdown in the comments](https://about.gitlab.com/images/blogimages/mr-reviews-with-vs-code/mr-review-long-comment.png){: .shadow.medium.center}\nVS Code supports Markdown in the comments\n{: .note .text-center}\n\n## What's next\n\nOver the next few milestones, we plan to make the commenting as interactive as you know it from the GitLab web interface. We'll start with editing existing comments, adding emoji reactions and resolving discussion threads. Lastly, we'll implement the full review functionality with creating comments and reviews[^4]. Each [iteration](https://handbook.gitlab.com/handbook/values/#iteration) will make the feature a bit more useful.\n\nI'm excited about the potential to stay in my editor for both creating and reviewing merge requests. I'm already using the current merge request review feature to get the initial understanding of what the MR tries to achieve. I can explore the related code more quickly in my editor. If you'd like to help us build the code review feature or just look at the current state of development, visit the [Merge Request Review epic](https://gitlab.com/groups/gitlab-org/-/epics/4607).\n\nYou can check out a walkthrough our initial proof of concept of merge request reviews in VS Code below:\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/kKA6i8oqZAA\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n[^1]: [IntelliJ](https://plugins.jetbrains.com/plugin/7447-gitlab-integration-plugin), [Atom](https://atom.io/packages/search?q=gitlab), [vim](https://github.com/shumphrey/fugitive-gitlab.vim), [Emacs](https://github.com/nlamirault/emacs-gitlab), ...\n[^2]: [Jira and Bitbucket](https://marketplace.visualstudio.com/items?itemName=Atlassian.atlascode), [GitHub Pull Requests and Issues](https://marketplace.visualstudio.com/items?itemName=GitHub.vscode-pull-request-github)\n[^3]: You can work around that by using the MR overview and commenting there.\n[^4]: [MR review: interacting with existing comments - POC](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/269) and [MR review: new comments and reviews POC](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/293) represent the initial investigation.\n\n[Cover image](https://art.ljubicapetkovic.com/cc-licensed/) by [Ljubica Petkovic](https://art.ljubicapetkovic.com), licensed under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\n{: .note}\n",[809,232,9],{"slug":3457,"featured":6,"template":700},"mr-reviews-with-vs-code","content:en-us:blog:mr-reviews-with-vs-code.yml","Mr Reviews With Vs Code","en-us/blog/mr-reviews-with-vs-code.yml","en-us/blog/mr-reviews-with-vs-code",{"_path":3463,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3464,"content":3470,"config":3475,"_id":3477,"_type":13,"title":3478,"_source":15,"_file":3479,"_stem":3480,"_extension":18},"/en-us/blog/oidc",{"title":3465,"description":3466,"ogTitle":3465,"ogDescription":3466,"noIndex":6,"ogImage":3467,"ogUrl":3468,"ogSiteName":685,"ogType":686,"canonicalUrls":3468,"schema":3469},"Secure GitLab CI/CD workflows using OIDC JWT on a DevSecOps platform","Learn a new method to authenticate using JWT to increase the security of CI/CD workflows.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667094/Blog/Hero%20Images/container-security.jpg","https://about.gitlab.com/blog/oidc","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Secure GitLab CI/CD workflows using OIDC JWT on a DevSecOps platform\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Dov Hershkovitch\"}],\n        \"datePublished\": \"2023-02-28\",\n      }",{"title":3465,"description":3466,"authors":3471,"heroImage":3467,"date":3472,"body":3473,"category":1119,"tags":3474},[3066],"2023-02-28","\n\nSecuring CI/CD workflows can be challenging. This blog post walks you through the problem validation, explores the JWT token technology and how it can be used with OIDC authentication, and discusses implementation challenges with authorization realms. You will learn about the current possibilities and future plans with GitLab 16.0. \n\n### Variables vs. secrets\nVariables are an efficient way to control and inject parameters into your jobs and pipelines, making managing and configuring the CI/CD workflows easier. You can read more about [how to use CI/CD variables](https://about.gitlab.com/blog/demystifying-ci-cd-variables/). An extra layer of security on top of variables to mask and protect, for now, is our “best-effort” to prevent sensitive variables from being accidentally revealed. However, variables are not a drop-in replacement for secrets. [Securing secrets natively](https://gitlab.com/gitlab-org/gitlab/-/issues/217355) is a solution that GitLab aspires to provide. Meanwhile, we recommend storing sensitive information in a dedicated secrets management solution. As a company, we will provide you abilities to integrate and retrieve secrets as part of your CI/CD workflows. \n\n## Security shifting left\nSensitive information like passwords, secret tokens, or shared IDs required to access tools and platforms need to be securely stored. They must also be highly available to their owners and the teams who use them. There are various secrets management solutions and frameworks available. They have addressed one problem but created new problems. For example: \"Which tool is right for our needs?\" More importantly, in software development: \"What's the best way to integrate this into our DevOps processes so that we're secure but still operating as efficiently as possible?\" Ignoring the security protocols in your organization is not an option. However, sensitive information should be stored as securely as possible. Something as simple as an access token stored in plain text can lead to security leaks and business incidents in the worst-case scenarios.\n\n## Initial support for JWT\nThe [JSON Web Token (JWT)](https://en.wikipedia.org/wiki/JSON_Web_Token) aims to build the integration bridge as an open standard for security claims exchange. It is a signed, short-lived, contextualized token that allows everyone to implement authentication between different products securely. The JWT consists of three parts: a header, a payload, and a signature.\n\n- The header represents the type of the token and the encryption algorithm.\n- The signature ensures that the token hasn't been altered.\n- The payload comprises a series of claims representing the information exchanged between two parties, which includes information about a GitLab user (ID, email, login) and the pipeline information (pipeline ID, job ID, environment, and more).\n\n_Example of GitLab JWT payload_\n\n```\n{\n  \"jti\": \"c82eeb0c-5c6f-4a33-abf5-4c474b92b558\",\n  \"iss\": \"gitlab.example.com\",\n  \"iat\": 1585710286,\n  \"nbf\": 1585798372,\n  \"exp\": 1585713886,\n  \"sub\": \"job_1212\",\n  \"namespace_id\": \"1\",\n  \"namespace_path\": \"mygroup\",\n  \"project_id\": \"22\",\n  \"project_path\": \"mygroup/myproject\",\n  \"user_id\": \"42\",\n  \"user_login\": \"myuser\",\n  \"user_email\": \"myuser@example.com\",\n  \"pipeline_id\": \"1212\",\n  \"pipeline_source\": \"web\",\n  \"job_id\": \"1212\",\n  \"ref\": \"auto-deploy-2020-04-01\",\n  \"ref_type\": \"branch\",\n  \"ref_protected\": \"true\",\n  \"environment\": \"production\",\n  \"environment_protected\": \"true\"\n}\n```\nUsing this information (called \"claims\"), you can implement an authentication condition where the token will get rejected if one of those claims does not match. You can use this to restrict access to only the authorized users and jobs in your pipelines.\n\nGitLab 12.10 added [initial support for JWT token-based connections](https://about.gitlab.com/releases/2020/04/22/gitlab-12-10-released/#retrieve-cicd-secrets-from-hashicorp-vault), which was later [enhanced](https://about.gitlab.com/releases/2020/09/22/gitlab-13-4-released/#use-hashicorp-vault-secrets-in-ci-jobs) with the `secrets:` keyword, as well as the `CI_JOB_JWT` predefined CI/CD variable, which is automatically injected into every job in a pipeline. This implementation was restricted to Hashicorp Vault, and users can use it to read secrets directly from the vault as part of their CI/CD workflow.\n \n### OIDC (JWT Version 2)\nThe logic we used to build the initial support for JWT opened up the possibility of connecting to other providers as well, but the first iteration was still restricted to Hashicorp Vault users.\n\nThis problem was addressed in GitLab 14.7 when we [released](https://about.gitlab.com/releases/2022/01/22/gitlab-14-7-released/#openid-connect-support-for-gitlab-cicd) the first \"Alpha\" version of JWT V2, which provided [Open ID Connect (OIDC)](https://openid.net/connect/) support for CI/CD.\n\nOIDC is an identity layer implemented on top of the JSON web token. You can securely authenticate against many products and services that implement OIDC, including AWS, GCP, and many more, making better use of the token's potential. Similar to our first JWT iteration, we added another [predefined CI/CD variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) `CI_JOB_JWT_V2` which is also automatically injected into every job in a CI/CD pipeline.\n\n### Securely store your secrets \nYour software supply chain should include everything needed to deliver and run your software. Securing your supply chain means you need to secure your software and the surrounding (cloud-native) infrastructure. In [GitLab 15.9](https://about.gitlab.com/releases/2023/02/22/gitlab-15-9-released/), we've added additional layers of protection to move our OIDC token from an Experiment to General Availability, increasing the security of your CI/CD workflows. \n\n\n#### Opt-in JWT token\nJSON web tokens (V1 and V2) are stored in CI/CD variables, which are injected automatically into all jobs in a CI/CD pipeline. However, it is likely most jobs in your pipeline do not need the token. In addition to the inefficiency of injecting unused tokens into all jobs in a pipeline, there is a potential security vulnerability. All it takes is one compromised job for this token to be leaked and used by an attacker to retrieve sensitive information from your organization. To minimize this risk, we've added the ability to restrict the token variable from all jobs in your pipeline and expose it only to the specific jobs that need it.\n\nTo declare the JSON web token in a job that needs it, configure the job in the `.gitlab-ci.yml` configuration file following this example:\n\n```yaml\njob_name:\n  id_token:\n    MY_JOB_JWT: # or any other variable name\n  ...\n```\n\nYou can minimize the token exposure across your pipeline, but ensure it is available to the jobs that require it.\n\n#### Audience claim (`aud:`)\nClaims constitute the payload part of a JSON web token and represent a set of information exchanged between two parties. The JWT standard distinguishes between reserved, public, and private claims.\n\nThe audience (`aud:`) claim is a reserved claim, which identifies the audience that the JWT is intended for (the target of the token). In other words, which services, APIs, or products should accept this token. If the audience claim does not match, the token is rejected, so the audience claim is an essential part of software supply chain security.\n\nThe option to configure the audience claim is done in the CI/CD configuration when [declaring the usage of the JWT token](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#id-tokens), if we'll continue from the previous example:\n\n```yaml\njob_name:\n  id_token:\n    MY_JOB_JWT: # or any other variable name\n        aud: \"...\" # mandatory field\n  script:\n    - my-authentication-script.sh MY_JOB_JWT….. # use the declared variables in a script\n  ```\n\nConfiguring the audience claim is mandatory for Vault users that leverage the [GitLab/Vault native integration](https://docs.gitlab.com/ee/ci/secrets/#use-vault-secrets-in-a-ci-job) (using the 'secrets:' keyword).\n\n```yaml\njob_name:\n  secrets:\n    VAULT_JWT_1: # or any other variable name\n      id_token:\n        aud: 'devs' # audience claim configuration\n    STAGING_DATABASE_PASSWORD: # VAULT_JWT_1 is the token to be used\n      vault: staging/db/password@ops\n```\n\n### Breaking changes and backward compatibility \nWe understand the increasing demand to secure your software supply chain. We recognize that many of our current users already use the JWT in what will soon be the \"old JWT method\" (V1). To mitigate this conflict, we've decided that moving to the new (OIDC) JWT method is optional until the next major release (GitLab 16.0). To use the new (OIDC) token, users must opt-in to this change from the UI settings and update the pipeline configuration, as explained in the previous sections. Users can continue using the Experiment or the \"old method\" until GitLab 16.0. (At that point, only the \"new\" (OIDC) JWT token and method will be available.)\n\nSeveral breaking changes were announced for both [Vault users](https://docs.gitlab.com/ee/update/deprecations.html#hashicorp-vault-integration-will-no-longer-use-ci_job_jwt-by-default) and [users of the JWT \"old\" methods](https://docs.gitlab.com/ee/update/deprecations.html#old-versions-of-json-web-tokens-are-deprecated). Those changes are scheduled for GitLab 16.0.\n\n## Three ways to use the JWT token\nThere are three ways to use a JWT to authenticate against different products in your CI/CD pipeline:\n- The \"old\" method, using the `secrets:` keyword and the `CI_JOB_JWT` variable, which is mainly used to integrate with Hashicorp Vault.\n- An \"Alpha\" version that uses the `CI_JOB_JWT_V2` OIDC token to integrate with different cloud providers.\n- A production-ready OIDC token, which is a secured version of the `CI_JOB_JWT_V2` token, used to authenticate with a variety of different products, like Vault, GCP, AWS, and so on.\n\nAll three methods are available until the next major version (GitLab 16.0). At that point, only the secured OIDC token will be available.\n\nTo prepare for this change, you should:\n\n1. Configure your pipelines to use the fully configurable and more secure [id_token](https://docs.gitlab.com/ee/ci/yaml/index.html#id_tokens) keyword.\n2. Enable the [Limit JSON Web Token (JWT) access setting](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#enable-automatic-id-token-authentication), which prevents the old tokens from being exposed to any jobs. (This setting will be permanently enabled for all projects in GitLab 16.0).\n3. If you use GitLab/Hashicorp native integration (using the [secrets:vault](https://docs.gitlab.com/ee/ci/yaml/#secretsvault) keyword), ensure the bound audience is prefixed with `https://`.\n\nThis should ensure a smooth transition to [GitLab 16.0](/upcoming-releases/) without breaking your existing workflows.\n\n\n",[9,851,787,788],{"slug":3476,"featured":6,"template":700},"oidc","content:en-us:blog:oidc.yml","Oidc","en-us/blog/oidc.yml","en-us/blog/oidc",{"_path":3482,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3483,"content":3489,"config":3495,"_id":3497,"_type":13,"title":3498,"_source":15,"_file":3499,"_stem":3500,"_extension":18},"/en-us/blog/pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes",{"title":3484,"description":3485,"ogTitle":3484,"ogDescription":3485,"noIndex":6,"ogImage":3486,"ogUrl":3487,"ogSiteName":685,"ogType":686,"canonicalUrls":3487,"schema":3488},"Pair GitLab and The Good Docs Project template to improve release notes","Creating compelling, detailed, human-readable notes for software releases is important. Using GitLab and this template from The Good Docs Project makes it easier.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099541/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_432673748_5xWPNsktdz2QChWhl16jGq_1750099540656.jpg","https://about.gitlab.com/blog/pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Pair GitLab and The Good Docs Project template to improve release notes\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Aaron Peters, Member, Good Docs Project\"}],\n        \"datePublished\": \"2024-01-23\",\n      }",{"title":3484,"description":3485,"authors":3490,"heroImage":3486,"date":3492,"body":3493,"category":740,"tags":3494},[3491],"Aaron Peters, Member, Good Docs Project","2024-01-23","Release notes allow software users to quickly understand the changes that come with the latest version of software. They also allow software publishers to highlight changes as important, or provide crucial information about the impact an upgrade may have. Some tools allow developers to \"generate\" release notes based on sources of data (such as completed items in DevOps systems), but notes produced this way tend to simply list changes without context. Writing release notes, however, provides teams with the opportunity to \"tell the story\" of the changes the new software version will bring.\n\nThough this process certainly requires a greater investment of time than publishing a basic changelog does, your users will certainly appreciate the results: release notes that explain the key elements of the release (such as new features, improvements, and known issues) in a well-organized, human-readable way.\n\n[The Good Docs Project's](https://thegooddocsproject.dev/welcome/) release notes template is designed to help you do exactly that. And the combination of GitLab's work management platform and our own [Release Notes template](https://gitlab.com/tgdp/templates/-/tree/main/release-notes?ref_type=heads) makes the job of putting out good, informative release notes easier.\n\n## The anatomy of quality release notes\n\nRelease notes that provide readers with a good picture of the version's changes require two primary inputs:\n\n- **A list of the changes included in the release**\n  At The Good Docs Project, all the management of the work of our contributors occurs in GitLab. So it's easy to refer to our release plans to identify which additions and improvements were completed and included in the release.\n- **A description of those changes including reasoning, importance, and impact**\n  This is where our project's Release Notes template can assist. Rather than staring at a blank page, wondering where to start, users can begin to fill in our template step-by-step, adjusting to taste.\n\nWe'll walk through each of these steps in the following sections as they occurred when creating the release notes to [our recent Dragon release](https://gitlab.com/tgdp/templates/-/releases/v1.1.0).\n\n## Gathering a release's changes\n\nAt The Good Docs Project, we use GitLab features — including setting milestones, creating/assigning issues, and tagging releases — to get our work out into the community (our prior blog post here at GitLab describes this process). The platform allows our worldwide contributor base to easily discover new things to work on and update everyone on their progress once they select something. When the time comes to package a release, it brings the added benefit of a tidy list of issues included in the project at the time of release.\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176084/Blog/hxo08y06apkumwuwm80c.png\" alt=\"The Milestone screen in GitLab provides an easy-to-scan list of work included in the release\" width=\"100%\" height=\"auto\">\n\nWhen creating the release notes for our project's Dragon milestone, we reviewed all the items included in the **Closed** column on the Milestone screen. This allowed us to pick the most important changes to highlight, while leaving out issues that wouldn't significantly impact a user's experience.\n\n## Crafting the release notes\n\nEquipped with a list of all the key updates in the release, we start writing the release notes. Our project's [Release Notes template](https://gitlab.com/tgdp/templates/-/blob/main/release-notes/template-release-notes.md?ref_type=heads) provides a ready-made Markdown skeleton comprised of key sections based on our contributors' research and experience. The accompanying [usage guide](https://gitlab.com/tgdp/templates/-/blob/main/release-notes/guide-release-notes.md?ref_type=heads) and [example of the template in action](https://gitlab.com/tgdp/templates/-/blob/main/release-notes/example-release-notes.md?ref_type=heads) provides additional tips and suggestions for writing effective release notes. The latter references our **Chronologue** project, a fictional telescope and application that can see through time, which is naturally well-documented.\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176091/Blog/jcpfxjqb5jpidssm3jlr.png\" alt=\"The Release Notes template comes ready to populate with 'the story' of your latest release\" width=\"100%\" height=\"auto\">\n\nOf course, our template is simply a starting point. Teams should always feel free to add sections where they make sense, remove them where they don't, and make the style of it their own. For example, we left out the **Bug fixes** and **Known issues** sections in our latest Dragon release notes, instead focusing on the new additions and improvements this release brought.\n\n## Adding release notes to the release\n\nGitLab's build tools also make it easy to add our notes while actually creating the release. First, we tagged one of our project's commits, then created a release from the tag. On GitLab's **Releases > New** screen, we can copy and paste the Markdown we wrote to automatically format the release notes.\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176096/Blog/otwduhvokgnqclon4ugx.png\" alt=\"Our templates are already in Markdown format, so when it's time to paste them into the release it works automagically!\" width=\"100%\" height=\"auto\">\n\nAnd just like that our release notes are done. With the assistance of the template, they required just an hour to write. And after an additional half-hour of work creating the release, we're ready to send our work out to the community. Our experience using the combination of GitLab and our templates has made the process of shipping our templates a piece of cake.\n\nIf you'd like to check out our templates, feel free to browse [our GitLab project](https://gitlab.com/tgdp).\nOr visit our [community page](https://thegooddocsproject.dev/community/) to learn how to join us in leveling up the state of technical documentation.\n\n*The [GitLab Open Source Partners](https://go.gitlab.com/030Ue3) are building the future of open source on GitLab. [Connect with them](https://gitlab.com/gitlab-com/marketing/developer-relations/open-source-program/gitlab-open-source-partners) on Gitlab.com.*\n",[743,495,268,9],{"slug":3496,"featured":90,"template":700},"pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes","content:en-us:blog:pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes.yml","Pair Gitlab And The Good Docs Project Template To Improve Release Notes","en-us/blog/pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes.yml","en-us/blog/pair-gitlab-and-the-good-docs-project-template-to-improve-release-notes",{"_path":3502,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3503,"content":3509,"config":3515,"_id":3517,"_type":13,"title":3518,"_source":15,"_file":3519,"_stem":3520,"_extension":18},"/en-us/blog/parent-child-pipelines",{"title":3504,"description":3505,"ogTitle":3504,"ogDescription":3505,"noIndex":6,"ogImage":3506,"ogUrl":3507,"ogSiteName":685,"ogType":686,"canonicalUrls":3507,"schema":3508},"How to get started with Parent-child pipelines","We introduced improvements to pipelines to help scale applications and their repo structures more effectively. Here's how they work.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667040/Blog/Hero%20Images/parent_pipeline_graph.png","https://about.gitlab.com/blog/parent-child-pipelines","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to get started with Parent-child pipelines\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Chris Ward\"}],\n        \"datePublished\": \"2020-04-24\",\n      }",{"title":3504,"description":3505,"authors":3510,"heroImage":3506,"date":3512,"body":3513,"category":785,"tags":3514},[3511],"Chris Ward","2020-04-24","As applications and their repository structures grow in complexity, a\nrepository `.gitlab-ci.yml` file becomes difficult to manage, collaborate\non, and see benefit from. This problem is especially true for the\nincreasingly popular \"[monorepo](https://en.wikipedia.org/wiki/Monorepo)\"\npattern, where teams keep code for multiple related services in one\nrepository. Currently, when using this pattern, developers all use the same\n`.gitlab-ci.yml` file to trigger different automated processes for different\napplication components, likely causing merge conflicts, and productivity\nslowdown, while teams wait for \"their part\" of a pipeline to run and\ncomplete.\n\n\nTo help large and complex projects manage their automated workflows, we've\nadded two new features to make pipelines even more powerful: Parent-child\npipelines, and the ability to generate pipeline configuration files\ndynamically.\n\n\n## Meet Parent-child pipelines\n\n\nSo, how do you solve the pain of many teams collaborating on many\ninter-related services in the same repository? \n\nLet me introduce you to [Parent-child\npipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html),\nreleased with with [GitLab\n12.7](/releases/2020/01/22/gitlab-12-7-released/#parent-child-pipelines).\nSplitting complex pipelines into multiple pipelines with a parent-child\nrelationship can improve performance by allowing child pipelines to run\nconcurrently. This relationship also enables you to compartmentalize\nconfiguration and visualization into different files and views. \n\n\n### Creating a child pipeline\n\n\nYou trigger a child pipeline configuration file from a parent by including\nit with the `include` key as a parameter to the `trigger` key. You can name\nthe child pipeline file whatever you want, but it still needs to be valid\nYAML.\n\n\nThe parent configuration below triggers two further child pipelines that\nbuild the Windows and Linux version of a C++ application. \n\n\n```cpp\n\n#include \u003Ciostream>\n\nint main()\n\n{\n  std::cout \u003C\u003C \"Hello GitLab!\" \u003C\u003C std::endl;\n  return 0;\n}\n\n```\n\n\nThe setup is a simple one but hopefully illustrates what is possible.\n\n\n```yaml\n\nstages:\n  - triggers\n\nbuild_windows:\n  stage: triggers\n  trigger:\n    include: .win-gitlab-ci.yml\n  rules:\n    - changes:\n      - cpp_app/*\n\nbuild_linux:\n  stage: triggers\n  trigger:\n    include: .linux-gitlab-ci.yml\n  rules:\n    - changes:\n      - cpp_app/*\n```\n\n\nThe important values are the `trigger` keys which define the child\nconfiguration file to run, and the parent pipeline continues to run after\ntriggering it. You can use all the normal sub-methods of `include` to use\nlocal, remote, or template config files, up to a maximum of three child\npipelines.\n\n\nAnother useful pattern to use for parent-child pipelines is a `rules` key to\ntrigger a child pipeline under certain conditions. In the example above, the\nchild pipeline only triggers when changes are made to files in the _cpp_app_\nfolder.\n\n\nThe Windows build child pipeline (`.win-gitlab-ci.yml`) has the following\nconfiguration, and unless you want to trigger a further child pipeline, it\nfollows standard a configuration format:\n\n\n```yaml\n\nimage: gcc\n\nbuild:\n  stage: build\n  before_script:\n    - apt update && apt-get install -y mingw-w64\n  script:\n    - x86_64-w64-mingw32-g++ cpp_app/hello-gitlab.cpp -o helloGitLab.exe\n  artifacts:\n    paths:\n      - helloGitLab.exe\n```\n\n\nDon't forget the `-y` argument as part of the `apt-get install` command, or\nyour jobs will be stuck waiting for user input.\n\n\nThe Linux build child pipeline (`.linux-gitlab-ci.yml`) has the following\nconfiguration, and unless you want to trigger a further child pipeline, it\nfollows standard a configuration format:\n\n\n```yaml\n\nimage: gcc\n\nbuild:\n  stage: build\n  script:\n    - g++ cpp_app/hello-gitlab.cpp -o helloGitLab\n  artifacts:\n    paths:\n      - helloGitLab\n```\n\n\nIn both cases, the child pipeline generates an artifact you can download\nunder the _Job artifacts_ section of the Job result screen.\n\n\nPush all the files you created to a new branch, and for the pipeline result,\nyou should see the two jobs and their subsequent child jobs.\n\n\n![Parent-child pipeline\nresult](https://about.gitlab.com/images/blogimages/non-dynamic-pipelines.png){:\n.shadow.medium.center}\n\nThe result of a parent-child pipeline\n\n{: .note.text-center}\n\n\n## Dynamically generating pipelines\n\n\nTaking Parent-child pipelines even further, you can also dynamically\ngenerate the child configuration files from the parent pipeline. Doing so\nkeeps repositories clean of scattered pipeline configuration files and\nallows you to generate configuration in your application, pass variables to\nthose files, and much more.\n\n\nLet's start with the parent pipeline configuration file:\n\n\n```yaml\n\nstages:\n  - setup\n  - triggers\n\ngenerate-config:\n  stage: setup\n  script:\n    - ./write-config.rb\n    - git status\n    - cat .linux-gitlab-ci.yml\n    - cat .win-gitlab-ci.yml\n  artifacts:\n    paths:\n      - .linux-gitlab-ci.yml\n      - .win-gitlab-ci.yml\n\ntrigger-linux-build:\n  stage: triggers\n  trigger:\n    include:\n      - artifact: .linux-gitlab-ci.yml\n        job: generate-config\n\ntrigger-win-build:\n  stage: triggers\n  trigger:\n    include:\n      - artifact: .win-gitlab-ci.yml\n        job: generate-config\n```\n\n\nDuring our self-defined `setup` stage the pipeline runs the\n`write-config.rb` script. For this article, it's a Ruby script that writes\nthe child pipeline config files, but you can use any scripting language. The\nchild pipeline config files are the same as those in the non-dynamic example\nabove. We use `artifacts` to save the generated child configuration files\nfor this CI run, making them available for use in the child pipelines\nstages.\n\n\nAs the Ruby script is generating YAML, make sure the indentation is correct,\nor the pipeline jobs will fail.\n\n\n```ruby\n\n#!/usr/bin/env ruby\n\n\nlinux_build = \u003C\u003C~YML\n    image: gcc\n    build:\n        stage: build\n        script:\n            - g++ cpp_app/hello-gitlab.cpp -o helloGitLab\n        artifacts:\n            paths:\n                - helloGitLab\nYML\n\n\nwin_build = \u003C\u003C~YML\n    image: gcc\n    build:\n        stage: build\n        before_script:\n            - apt update && apt-get install -y mingw-w64\n        script:\n            - x86_64-w64-mingw32-g++ cpp_app/hello-gitlab.cpp -o helloGitLab.exe\n        artifacts:\n            paths:\n                - helloGitLab.exe\nYML\n\n\nFile.open('.linux-gitlab-ci.yml', 'w'){ |f| f.write(linux_build)}\n\nFile.open('.win-gitlab-ci.yml', 'w'){ |f| f.write(win_build)}\n\n```\n\n\nThen in the `triggers` stage, the parent pipeline runs the generated child\npipelines much as in the non-dynamic version of this example but instead\nusing the saved `artifact` files, and the specified `job`.\n\n\nPush all the files you created to a new branch, and for the pipeline result,\nyou should see the three jobs (with one connecting to the two others) and\nthe subsequent two children.\n\n\n![Dynamic parent-child pipeline\nresult](https://about.gitlab.com/images/blogimages/dynamic-pipelines.png){:\n.shadow.medium.center}\n\nThe result of a dynamic parent-child pipeline\n\n{: .note.text-center}\n\n\n## Pipeline flexibility\n\n\nThis blog post showed some simple examples to give you an idea of what you\ncan now accomplish with pipelines. With one parent, multiple children, and\nthe ability to generate configuration dynamically, we hope you find all the\ntools you need to [build CI/CD workflows](/topics/ci-cd/) you need.\n\n\nYou can also watch a demo of Parent-child pipelines below:\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/n8KpBSqZNbk\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n",[108,696,1263,9],{"slug":3516,"featured":6,"template":700},"parent-child-pipelines","content:en-us:blog:parent-child-pipelines.yml","Parent Child Pipelines","en-us/blog/parent-child-pipelines.yml","en-us/blog/parent-child-pipelines",{"_path":3522,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3523,"content":3529,"config":3537,"_id":3539,"_type":13,"title":3540,"_source":15,"_file":3541,"_stem":3542,"_extension":18},"/en-us/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection",{"title":3524,"description":3525,"ogTitle":3524,"ogDescription":3525,"noIndex":6,"ogImage":3526,"ogUrl":3527,"ogSiteName":685,"ogType":686,"canonicalUrls":3527,"schema":3528},"Prevent secret leaks in source code with GitLab Secret Push Protection","Learn how Secret Push Protection, now generally available, adds to a defense-in-depth detection strategy and decreases the resources needed to remediate secret leaks.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097761/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%286%29_6vL96ttKF8zJLLqfPpvFs_1750097761137.png","https://about.gitlab.com/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Prevent secret leaks in source code with GitLab Secret Push Protection\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Amar Patel\"},{\"@type\":\"Person\",\"name\":\"Sara Meadzinger\"}],\n        \"datePublished\": \"2024-06-24\",\n      }",{"title":3524,"description":3525,"authors":3530,"heroImage":3526,"date":3533,"body":3534,"category":830,"tags":3535,"updatedDate":3536},[3531,3532],"Amar Patel","Sara Meadzinger","2024-06-24","Secret Push Protection is now generally available for all GitLab Ultimate and GitLab Dedicated customers. [Secret Push Protection](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/) blocks secrets such as keys and API tokens from being pushed to GitLab. The content of each commit is checked for [high-confidence secrets](https://docs.gitlab.com/ee/user/application_security/secret_detection/detected_secrets.html) when pushed to GitLab. If any high-confidence secrets are detected, the push is blocked. By protecting secrets from leaking in the first place, your team can greatly reduce risk and reduce time spent on rotating secrets.\n\n## The risk of leaked secrets\n\nSecrets, such as tokens and API keys, are frequently used by applications to authenticate and provide access to sensitive data. Developers sometimes inadvertently hardcode these secrets, and then push that code into source management systems, like GitLab. Hardcoded secrets stored in plain text are a low-effort, high-value target for malicious actors, as numerous recent high-profile breaches have demonstrated. Secrets do not require any special skills to exploit and many secrets do not automatically expire. Therefore, once a malicious actor has access to a secret, they can continue using it indefinitely to cause data breaches, service disruptions, IP theft, source code theft, and software supply chain compromises. Both [Verizon’s annual Data Breach Investigations Report](https://www.verizon.com/business/resources/reports/dbir) and [IBM’s annual Cost of a Data Breach report](https://www.ibm.com/reports/data-breach) have repeatedly reported that compromised credentials, which include secrets, are one of the most frequent and expensive source of breaches. \n\nIBM’s research also indicates that taking a DevSecOps, or shift-left, approach is the most effective way to reduce the average cost of a data breach. Until now, GitLab’s primary secret detection method has been [Pipeline Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/), which scans committed files after they have been pushed to GitLab and identifies secrets that are already leaked. Once a secret has leaked, it should be considered compromised and must be rotated according to the steps outlined by the secret issuer. Remediating detected secrets requires security teams and developers to work closely together to follow the steps outlined by a secret issuer to rotate the leaked secret. It can be a tedious, confusing, and risky process. Utilizing GitLab’s Secret Push Protection feature, you can shift secret detection further left, protect your secrets from leaking in the first place, and reduce the amount of time and energy required to remediate leaks.\n\n## How Secret Push Protection works\nOnce [Secret Push Protection](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/) is enabled on a project, developers are blocked from pushing code to projects that contain any high-confidence secrets. This ensures a performant experience when pushing your code and also results in a lower number of false alerts. **Note:** Here is the [list of high-confidence patterns Secret Push Protection supports](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/detected_secrets.html). \n\nWhile we are checking the contents of each commit, we've [excluded](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/#coverage) a number of factors in order to optimize the performance of this workflow. Because of this, we recommend using Secret Push Protection in a layered approach alongside [Pipeline Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline). Using both features in tandem maximizes coverage to identify more leaked secrets across the software development lifecycle.\n\n# Get started with Secret Push Protection\n\nWe've put a video playlist together to help you get started on using this feature:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/videoseries?si=kRG65YbljQ-Nu2wa&list=PL05JrBw4t0KoADm-g2vxfyR0m6QLphTv-\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Enable Secret Push Protection\n\nOn GitLab Dedicated and Self-managed, you must allow the use of Secret Push Protection in your instance and then enable it per project. On GitLab.com, you only need to enable it per project.\n\nYou must have at least the Maintainer role to enable push protection for the project.\n\n1. On the left sidebar, select **Search** or **Go to** and find your project.\n1. On the left sidebar, select **Secure > Security configuration**.\n1. Turn on the Secret Push Protection toggle.\n\n![secret push protection - toggle](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097769/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-10-18_at_12.24.47_PM_aHR0cHM6_1750097769198.png)\n\n## Skip push protection\n\nIn some instances, when a push is blocked, you might find it necessary to skip Secret Push Protection. For example, a developer may need to commit a placeholder secret for testing. You can skip Secret Push Protection via a Git option or commit message, meeting developers in whichever Git client they are using. \n\n## Add exclusions\n\nWe released exclusions, giving you flexibility to exclude certain paths, rules from the default ruleset, or raw values from being scanned, detected, and blocked by push protection. From the Security Configuration page, Maintainers and project Owners can manage push protection exclusion lists within the UI on a per-project basis. \n\n![secret push protection - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097769/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097769199.png)\n\n## Audit events\n\nDisabling Secret Push Protection, or even skipping it altogether, can prove to be costly if not done for the appropriate reasons. We've introduced [audit events](https://docs.gitlab.com/ee/user/compliance/audit_events.html) to help administrators and security teams understand where and how this feature is being used, and to assist in any secrets-related investigations.\n\nWe currently log audit events when Secret Push Protection is: \n\n- enabled/disabled at an instance level\n- enabled/disabled at project level\n- skipped via a push option\n- skipped via a commit message \n\nAnd when an exclusion is:\n- created\n- updated\n- deleted \n\n![secret push protection - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097769/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097769200.png)\n\nThese audit events can be used in conjunction with [audit event streaming](https://docs.gitlab.com/ee/administration/audit_event_streaming/) to manage audit logs in third-party systems (like SIEMs), enabling customers to capture trends such as: how many times push protection is being skipped; which projects frequently bypass push protection; and which secrets are commonly skipped and may need to be excluded moving forward. \n\n# Dogfooding Secret Push Protection\n\nWe [dogfood everything](https://about.gitlab.com/handbook/engineering/development/principles/#dogfooding) here at GitLab. We've [collaborated](https://gitlab.com/groups/gitlab-org/-/epics/13523) with various teams across the organization to enable this feature across key projects, including our primary GitLab codebase. This process has enabled us to identify and address improvements early in the development process, and it has increased our confidence in the stability, performance, and customer workflows for the release of this feature.\n\n# What's next\n\nYou can help us improve this feature by commenting on [this Secret Push Protection feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/467408). We’ll incorporate your feedback and make [additional improvements](https://gitlab.com/groups/gitlab-org/-/epics/13107) as we continue to add new capabilities to the feature.\n\n> Learn more about the [Secret Push Protection](https://docs.gitlab.com/ee/user/application_security/secret_detection/secret_push_protection/).\n\n# Read more\n\n- [How Secret Detection can proactively revoke leaked credentials](https://about.gitlab.com/blog/how-secret-detection-can-proactively-revoke-leaked-credentials) \n- [How to implement secret management best practices with GitLab](https://about.gitlab.com/the-source/security/how-to-implement-secret-management-best-practices-with-gitlab/)\n- [GitLab native secrets manager to give software supply chain security a boost](https://about.gitlab.com/blog/gitlab-native-secrets-manager-to-give-software-supply-chain-security-a-boost)",[830,9,696,495,694],"2024-10-17",{"slug":3538,"featured":6,"template":700},"prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection","content:en-us:blog:prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection.yml","Prevent Secret Leaks In Source Code With Gitlab Secret Push Protection","en-us/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection.yml","en-us/blog/prevent-secret-leaks-in-source-code-with-gitlab-secret-push-protection",{"_path":3544,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3545,"content":3551,"config":3556,"_id":3558,"_type":13,"title":3559,"_source":15,"_file":3560,"_stem":3561,"_extension":18},"/en-us/blog/provision-group-runners-with-google-cloud-platform-and-gitlab-ci",{"title":3546,"description":3547,"ogTitle":3546,"ogDescription":3547,"noIndex":6,"ogImage":3548,"ogUrl":3549,"ogSiteName":685,"ogType":686,"canonicalUrls":3549,"schema":3550},"Provision group runners with Google Cloud Platform and GitLab CI","This tutorial will teach you how to set up a new group runner on GitLab.com using Google Cloud Platform in less than 10 minutes.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098300/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_623844718_4E5Fx1Q0DHikigzCsQWhOG_1750098300048.jpg","https://about.gitlab.com/blog/provision-group-runners-with-google-cloud-platform-and-gitlab-ci","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Provision group runners with Google Cloud Platform and GitLab CI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sarah Matthies\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2024-11-19\",\n      }",{"title":3546,"description":3547,"authors":3552,"heroImage":3548,"date":3553,"body":3554,"category":785,"tags":3555},[1524,847],"2024-11-19","Are you interested in hosting your own servers to run your GitLab CI/CD\npipelines but don’t know where to begin? Setting up a GitLab Runner to run\nyour pipelines on your own infrastructure can seem like a daunting task as\nit requires infrastructure knowledge and the know-how to maintain that\ninfrastructure. Typically this process requires the provision of\ninfrastructure, the installing of dependency, and testing that it works with\nyour GitLab instance.\n\n\nThis article highlights how easy it is to easily spin up a GitLab Runner of\nyour own utilizing GitLab’s Google Cloud Integration. Follow this tutorial\nand it will teach you how to set up a new group runner on GitLab.com using\nGoogle Cloud Platform in less than 10 minutes!\n\n\nYou will learn how to:\n\n\n- Create a new group runner.\n\n- Configure the new group runner’s tags and description.\n\n- Register the new group runner by adding in configurations.\n\n- Provision the GitLab Runner utilizing `gcloud cli` and Terraform.\n\n- Have your GitLab Runner pick up its first GitLab CI job.\n\n\n## Prerequisites\n\n- A terminal with Bash installed\n\n- Owner access on a Google Cloud Platform project\n\n- Terraform (or OpenTofu) [Version\n1.5](https://releases.hashicorp.com/terraform/1.5.7/) or greater \n\n- [gcloud CLI](https://cloud.google.com/sdk/docs/install) \n\n- 10 minutes\n\n\n## Tutorial\n\n1. Create a new group runner under __Build > Runners > New Group Runner__.\n\n\n__Note:__ Navigate to the group level.\n\n\n![GitLab Runner setup\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750098317126.png)\n\n\n2. Configure the new group runner's tags, description, and any additional\nconfigurations.\n\n\n![New Group Runner\nsetup](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098317127.png)\n\n\n3. Select __Google Cloud__.\n\n\n![Select Google Cloud\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098317129.png)\n\n\n4. Copy your project ID from Google Cloud Platform.\n\n\n![Copy project ID from GCP\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098317131.png)\n\n\n5. Fill out your Google Cloud project ID and choose a region, zone, and type\nof machine you want to use.\n\n\n![Screen to fill out Google Cloud\ninformation](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750098317132.png)\n\n\n6\\. Once this information is filled out, click **Setup instructions**.\n\n\nRun the bash script provided in Step 1 above.\n\n\n**Note:** This script was saved to a file called `setup.sh` for ease of use.\nYou may copy this right into your terminal if you are confident in\ndebugging.\n\n\n![Setup instructions\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750098317134.png)\n\n\n![Script for GitLab\nRunner](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750098317135.png)\n\n\n7\\. Create a `main.tf` file and follow the instructions in GitLab.\n\n\n**Note:** If you want to use OpenTofu instead of Terraform, you can still\ncopy the code and only have to adjust the Terraform commands for applying\nthe configuration. \n\n\n![Install and register GitLab Runner\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750098317136.png)\n\n\nOnce successfully provisioned, you should be see the following:\n\n\n![GitLab Runner\ncode](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750098317137.png)\n\n\n8\\. If you close the instructions and click the **View runners** button, you\nwill now have a newly provisioned runner present with \"Never contacted\" as\nits status.\n\n\n![Newly provisioned runner on\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098317/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098317139.png)\n\n\n9\\. In any project, add the following `.gitlab-ci.yml`.\n\n\n```  \n\nstages:  \n  - greet\n\nhello_job:  \n  stage: greet  \n  tags:  \n    - gcp-runner  \n  script:  \n    - echo \"hello\"  \n```\n\n\nVolia! You have set up your first GitLab Runner utilizing Google Cloud\nPlatform.\n\n\n# Next steps\n\n\nNow that you have provisioned your very own GitLab Runner, consider\noptimizing it for your specific use case. Some things to consider with your\nrunner moving forward:\n\n\n- Is the runner I provisioned the right size? Does it need additional\nresources for my use case? \n\n- Does the GitLab Runner contain all the dependency my builds need?  \n\n- How can I store the GitLab Runner as infrastructure as code?\n\n\n> Make sure to bookmark the [Provisioning runners in Google Cloud\ndocumentation](https://docs.gitlab.com/ee/ci/runners/provision_runners_google_cloud.html)\nfor easy reference.\n",[9,495,787,108,1016,1264,232],{"slug":3557,"featured":6,"template":700},"provision-group-runners-with-google-cloud-platform-and-gitlab-ci","content:en-us:blog:provision-group-runners-with-google-cloud-platform-and-gitlab-ci.yml","Provision Group Runners With Google Cloud Platform And Gitlab Ci","en-us/blog/provision-group-runners-with-google-cloud-platform-and-gitlab-ci.yml","en-us/blog/provision-group-runners-with-google-cloud-platform-and-gitlab-ci",{"_path":3563,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3564,"content":3570,"config":3575,"_id":3577,"_type":13,"title":3578,"_source":15,"_file":3579,"_stem":3580,"_extension":18},"/en-us/blog/publishing-an-astro-site-with-pages",{"title":3565,"description":3566,"ogTitle":3565,"ogDescription":3566,"noIndex":6,"ogImage":3567,"ogUrl":3568,"ogSiteName":685,"ogType":686,"canonicalUrls":3568,"schema":3569},"How to publish your Astro Site with GitLab Pages","Learn how to deploy an Astro Site with GitLab Pages.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682473/Blog/Hero%20Images/shot-by-cerqueira-0o_GEzyargo-unsplash.jpg","https://about.gitlab.com/blog/publishing-an-astro-site-with-pages","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to publish your Astro Site with GitLab Pages\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Janis Altherr\"}],\n        \"datePublished\": \"2022-10-24\",\n      }",{"title":3565,"description":3566,"authors":3571,"heroImage":3567,"date":3572,"body":3573,"category":785,"tags":3574},[1563],"2022-10-24","Astro is an amazing new framework to create content-focused static sites and\nGitLab Pages is a great way to deploy a site built with Astro. Here's a\nstep-by-step guide on how to build and deploy an Astro Site with GitLab\nPages.\n\n\n## Create the project locally\n\n\nFirst, create the Astro Project locally using the Astro CLI.\n\n\nNote: Even though we're offering a [project\ntemplate](https://gitlab.com/pages/astro),\n\nwe recommend using the CLI locally to scaffold your project. This ensures\nyou can create your project with the latest defaults.\n\n\n```shell\n\nnpm create astro@latest\n\n```\n\n\nNow follow the CLI instructions. As part of the setup, Astro will create the\n\nproject folder for you. During the course of the setup Astro will ask\nwhether you'd like to initialize a new Git repository. Answer this with `y`\n(yes).\n\n\nOnce the Astro CLI is done scaffolding your project, `cd` into the new\nfolder:\n\n\n```shell\n\ncd \u003Cyour-project>\n\n```\n\n\n## Configure Astro for GitLab Pages\n\n\nAstro comes with a few defaults that are incompatible with GitLab Pages. So\nbefore continuing, we need to set up a compatible config.\n\nEdit your `astro.config.mjs` to include the following:\n\n\n```javascript\n\n// astro.config.mjs\n\nimport { defineConfig } from 'astro/config';\n\n\n// https://astro.build/config\n\nexport default defineConfig({\n  // GitLab Pages requires exposed files to be located in a folder called \"public\".\n  // So we're instructing Astro to put the static build output in a folder of that name.\n  outDir: 'public',\n\n  // The folder name Astro uses for static files (`public`) is already reserved\n  // for the build output. So in deviation from the defaults we're using a folder\n  // called `static` instead.\n  publicDir: 'static',\n});\n\n```\n\n\nWhy are we doing this? GitLab Pages is a way to publish some files in a\n\nrepository, no matter what build tool you used to generate them. Unlike with\n\nother deployment tools the exposed files and the source code can live \n\ntogether in one place. So to ensure you don't accidentally expose sensitive\n\nfiles we're requiring you to consciously put them into a\n\nfolder named \"public\".\n\n\nBy default, Astro uses `public` for something different – the static \n\nassets. So we have to change that behavior. The above config tells Astro\n\nthat we'll put the static files in a folder named `static` and want the\n_output_\n\nfiles to be put in a folder named, as required, `public`.\n\n\nAstro already generated that assets folder under the old name while\n\nscaffolding, so we'll have to rename it. Inside your Astro project folder,\nrun:\n\n\n```shell\n\nmv public static\n\n```\n\n\nDepending on your project configuration, GitLab Pages will deploy your site \n\nat a URL that follows the format simlar to `https://\u003Cuser-or-group>.gitlab.\n\nio/\u003Cproject-name>`. If you want to use the default URL, you need to adjust\nAstro\n\nto the fact that the site is not mounted at the root path, otherwise it may \n\nnot load static assets (such as the CSS files) correctly. \n\n\n[Visit the\ndocumentation](https://docs.gitlab.com/ee/user/project/pages/getting_started_part_one.html#gitlab-pages-default-domain-names)\n\nto find out the URL schema of the project you intend to create, then add the\n\nfollowing line to your `astro.config.mjs`. (Skip this step if you're\ncreating\n\na user or group page):\n\n\n```javascript\n\n// astro.config.mjs\n\nexport default defineConfig({\n  // ...\n  base: '/\u003Cproject-name>'\n  // In case the project is owned by a subgroup, use:\n  // base: '/\u003Csubgroup>/\u003Cproject-name>'\n});\n\n```\n\n\nAstro\n[recommends](https://docs.astro.build/en/reference/configuration-reference/#site) \n\nadding the final site's full URL to generate the sitemap, so add it now to\nyour\n\n`astro.config.mjs`:\n\n\n```javascript\n\n// astro.config.mjs\n\nexport default defineConfig({\n  // ...\n  site: 'https://\u003Cuser-or-group>.gitlab.io'\n  \n  // Note: Instead of specifying both `base` and `site`, you can simply\n  // use the full URL here:\n  // site: 'https://\u003Cuser-or-group>.gitlab.io/\u003Cproject-name>'\n  // or for pages owned by a subgroup:\n  // site: 'https://\u003Cgroup>.gitlab.io/\u003Csubgroup>/\u003Cproject-name>'\n});\n\n```\n\n\nNow that you've successfully configured your project, you can commit your\n\nchanges.\n\n\n```shell\n\ngit add -A\n\ngit commit -m \"Initial commit\"\n\n```\n\n\n## Set up the remote repository\n\n\nYou can't push the code as we have yet to set up the remote repository.\nVisit\n\nGitLab and create a new project. When asked, select \"Create blank project.\"\n\n\nIn the setup screen, select \"GitLab Pages\" as the deployment target. Choose\nthe\n\nvisibility level however you like. This is mainly asking whether your\nsource \n\ncode is public, although it does affect the initial setting (see \"Making a \n\nprivate project's site public\" below).\n\n\nMake sure you unset the checkbox next to \"Initialize repository with a\nREADME\",\n\notherwise GitLab will begin a new Git history that you will have to\nreconcile\n\nwith your existing local one.\n\n\nOnce the Project is set up, follow the instructions on how to add an\n_existing\n\nrepository_ – if you don't have an existing remote, so you can just run:\n\n\n```shell\n\ngit remote add origin \u003Cgit-project-url>\n\ngit push -u origin --all\n\n```\n\n\nNow you've synced your local code with Gitlab, let's finish publishing it\nwith\n\nPages.\n\n\n## Create a Pages pipeline\n\n\nIn GitLab, go to your project's settings and select Pages. You will be\nwelcomed\n\nby a screen that helps you build a `.gitlab-ci.yml` file.\n\n\n![Screenshot: The \"Get stated with Pages\"\nUI](https://about.gitlab.com/images/blogimages/astro-pages/wizard_step_1.png)\n\n\nEnter \"node:lts\" as the build image. This will give you the latest node \n\nenvironment with long-time support.\n\n\nWe've already configured Astro to output our files in a folder named\n`public`,\n\nso you can check the checkbox asking you to confirm this.\n\n\nOn the next page, enter `npm ci` as the installation step. Running `npm ci` \n\ninstead of `npm install` is recommended for CI environments such as GitLab\n\nPipelines as it uses the `package-lock.json` to match the installed version \n\nwith the one you used during development. See the [npm\ndocumentation](https://docs.npmjs.com/cli/v8/commands/npm-ci)\n\nto learn more about `npm ci`.\n\n\n![Screenshot: Inputting the installation\nstep](https://about.gitlab.com/images/blogimages/astro-pages/wizard_step_2.png)\n\n\nOn the last page, enter the build command \"npm run build\". Again, click\n\"next\".\n\n\n![Screenshot: Inputting the build\nstep](https://about.gitlab.com/images/blogimages/astro-pages/wizard_step_3.png)\n\n\nNext to the inputs you see the pipeline file that has been built for you. \n\nThis is the one we want to add to the repository to enable Pages.\n\n\n![Screenshot: The finished file and the commit\nstep](https://about.gitlab.com/images/blogimages/astro-pages/wizard_step_4.png)\n\n\nHow does it work in detail? If GitLab sees a job named `pages`, it will \n\nlook for artifacts inside a root folder `public` and then create a \n\nGitLab Pages deployment from it.\n\n\nThe `rules` section ensures the pages deployment is only triggered by \n\ncommits to the default branch. Every time you push a change to your default \n\nbranch, Pages will publish the new changes. \n\n\nIf you're happy with the pipeline, enter a commit message and click\n\"commit\".\n\n(Make sure you run `git pull` locally before doing any more changes to \n\nprevent issues with diverging histories.)\n\n\nNow having added a commit with a `.gitlab-ci.yml` file, GitLab has kicked\noff\n\na pipeline. Visit CI/CD > Pipelines to see the progress. After a couple of \n\nminutes, you should see the pipeline has succeeded. (If it's showing\n\"failed\", \n\nclick on the status button to see the job logs.)\n\n\n![Screenshot:\nPipelines](https://about.gitlab.com/images/blogimages/astro-pages/pipeline_overview.png)\n\n\nOnce the pipeline has completed, go back to Settings > Pages. You should now\nsee\n\nthe various settings of your site, including your new site's URL. Click on \n\nit and, congratulations, you've just deployed your Astro Site wit GitLab \n\nPages!\n\n\n![Screenshot: The deployed\npage](https://about.gitlab.com/images/blogimages/astro-pages/deployed_site.png)\n\n\n## Making a private project's site public\n\n\nBy default, a private project's Pages site is only accessible to project \n\nmembers. If you want your source code to be private, but still have a\npublic \n\nsite, go to Settings/General and expand \"visibility, project features,\npermissions\", scroll down to \"Pages\" and set \n\nit to \"Everyone\".\n\n\n## Keep reading\n\n\n- [Tutorial: Use the GitLab UI to deploy your static\nsite](https://docs.gitlab.com/ee/user/project/pages/getting_started/pages_ui.html)\n\n- [Astro Docs: Deploy your Astro Site to GitLab\nPages](https://docs.astro.build/en/guides/deploy/gitlab/)\n\n- [Watch a video on how to create a Pages Pipeline with the\nWizard](https://youtu.be/49hgxqPGofw)\n",[9,232,694],{"slug":3576,"featured":6,"template":700},"publishing-an-astro-site-with-pages","content:en-us:blog:publishing-an-astro-site-with-pages.yml","Publishing An Astro Site With Pages","en-us/blog/publishing-an-astro-site-with-pages.yml","en-us/blog/publishing-an-astro-site-with-pages",{"_path":3582,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3583,"content":3588,"config":3594,"_id":3596,"_type":13,"title":3597,"_source":15,"_file":3598,"_stem":3599,"_extension":18},"/en-us/blog/publishing-obsidian-notes-with-gitlab-pages",{"title":3584,"description":3585,"ogTitle":3584,"ogDescription":3585,"noIndex":6,"ogImage":2238,"ogUrl":3586,"ogSiteName":685,"ogType":686,"canonicalUrls":3586,"schema":3587},"Publishing Obsidian.md notes with GitLab Pages","How to publish your Obsidian.md documents to a GitLab Pages site","https://about.gitlab.com/blog/publishing-obsidian-notes-with-gitlab-pages","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Publishing Obsidian.md notes with GitLab Pages\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Scott Hampton\"}],\n        \"datePublished\": \"2022-03-15\",\n      }",{"title":3584,"description":3585,"authors":3589,"heroImage":2238,"date":3591,"body":3592,"category":785,"tags":3593},[3590],"Scott Hampton","2022-03-15","[Obsidian.md](https://obsidian.md) is a \"knowledge base\" application that\nuses plain text Markdown files stored in a local folder to\n\norganize your notes. The product has been growing in popularity, partly\nbecause of how extensible it is. There are a\n\nlot of community built plugins to help users configure the application to\nsupport their specific workflow. There are\n\nmany people that use Obsidian to write their blog posts. [Obsidian offers a\npaid service to publish your notes directly](https://obsidian.md/publish)\n\nand is completely compatible with features Obsidian offers. I suggest you\nsupport the Obsidian developers if their product\n\nworks for you. If you are looking for an alternative way to publish, this\nblog post provides a tutorial for how to publish your notes using GitLab\n\nPages.\n\n\nYou can find an Obsidian.md example in [this demonstration\nproject](https://gitlab.com/gitlab-org/frontend/playground/obsidian-and-gitlab-pages-demo)\n\nwhich deploys [a GitLab Pages\nsite](https://gitlab-org.gitlab.io/frontend/playground/obsidian-and-gitlab-pages-demo/). \n\n\n## What is Obsidian markdown?\n\n\nObsidian is markdown-based system, which means it incorporates tags, plugins\nand backlinks to create an easy-to-use system. It makes it possible for you\nto use symbols inside the text that are interpreted as text formatting. This\n[link](https://www.markdownguide.org/cheat-sheet/) is a cheat sheet of all\nthe mardown syntax elements.\n\n\n### Benefits of Obsidian.md\n\n\nPerhaps the most significant benefit of Obsidian markdown (md) is its\nsimple, straightforward design and the excellent support provided. It is\nalso extensible, with plenty of community plugins available. \n\n\nThere is no proprietary formatting, encoding. This gives you greater control\nover how you backup files and manage change tracking.\n\n\nObsidian doesn't support git right out the box, it requires a community\nplugin called Obsidian Git. However, one the plugin is installed, “you end\nup with the greatest change tracking/archiving tool at your disposal,” one\nuser\n[raves](https://www.faesel.com/blog/why-every-developer-needs-to-use-obsidian).\n\n\n### How is Obsidian.md different from other markdown languages?\n\n\nObsidian markdown [differs from other markdown\neditors](https://cylab.be/blog/149/what-is-obsidianmd-and-why-you-should-use-it)\nin that it uses the “Linked Thought” feature, which refers to a group of\nnote-taking applications that allow you to link thoughts and notes together\nseamlessly. Because it is based on the [Markdown\nlanguage](https://en.wikipedia.org/wiki/Markdown), it is light-weight. The\ntool expands on the markdown language with additional functionality, such as\ncreating links between files, offering \"hover over preview\" of links and\neasy inclusion and management of sources.\n\n\nFor example, Obsidian lets you hover over any links added to a document and\nsee a small preview of what the links refers to. You just need to position\nyour mouse over the \"Format your notes\" link.\n\n\n### Some notable features of Obsidian.md\n\n\nThere’s a visually-striking graph view that’s acts as a map of all your\nfiles stored in Obsidian. There is also a markdown format importer that can\nfind and replace certain Markdown syntax elements in your files, and support\nfor [math and\ndiagram](https://publish.obsidian.md/help/How+to/Format+your+notes) syntax.\n\n\nAlso noteworthy is that Obsidian makes it easy to publish notes online and\nit stores all of your files in plaintext markdown files.\n\n\nObsidian supports CommonMark and GitHub Flavored Markdown (GFM) so you can\nembed notes and other files. It stores data in folders of markdown files so\nyou can access your notes with other text editors or markdown apps. Obsidian\nalso lets you open existing folders of markdown files.\n\n\n## Is Obsidian good for notes?\n\n\nObsidian is a very capable, free note-taking app (with advanced, paid tiers\navailable as well). It touts itself as a [“second\nbrain”](https://obsidian.md/) that is good for creating a knowledge base,\nmarkdown file editor and linking notes together. It is designed to take\nnotes quickly and is easy to use, making it an ideal app. You just open the\napp, create a new note and start typing.\n\n\nIt works across multiple platforms, including Windows, iOS, Android and\nLinux.\n\n\nObsidian has been called the [“most advanced note-taking\napp.”](https://deskoflawyer.com/secure-note-taking-apps/)\n\n\n## Setting up Obsidian notes\n\n\nOnce you download the app, you will see the main Obsidian window, which has\nthe different options on the left, then the folder/files panel and the\ncomposition area where you an create and edit your notes.\n\n\nThere are four icons on the left side: collapse panel, open quick switcher,\nopen graph view, and open markdown importer. The collapse panel shows (or\nhides) the left panel.When you tap the open quick switcher button, it brings\nup a text box where you can begin to type. The open graph view shows a graph\nlisting the connections each page has. The open markdown importer lets you\nimport markdown files into Obsidian from other applications.\n\n\nYou’ll also see three buttons: \n\n\n1. Open another vault \n\n2. Help\n\n3. Settings\n\n\nThe vault refers to a collection of notes that you can open or create.\n\n\nYou have the option of either creating a note directly or creating a note\nvia a link. In the former instance, in the folder panel, click on the “new\nnote” button or use the keyboard shortcut for Windows: Control N, or for\nMac: Command N. Now you’ve created a new note.\n\n\nAn interesting time-saving feature is that you can create a note via a link\nand assign a name to that new note. You have to click on the link to\nactually create it.\n\n\nYou can find a helpful guide\n[here](https://www.sitepoint.com/obsidian-beginner-guide/).\n\n\n## Organizing an Obsidian note using folders\n\n\nWhen you begin using Obsidian you have to designate where you want to keep\nyour notes. If you already have your notes in markdown format in a folder,\nyou would choose the “open folder as vault” option. Otherwise, you can\ncreate a new vault and choose a location to store your notes.\n\n\nYou can drag and drop notes to move them around. There are three icons at\nthe top pane that allow you to create a new note, make a new folder, or\nchange the sorting order.\n\n\nObsidian has a powerful search feature that checks the content of your notes\nand returns all results very quickly. Access it by clicking on the\nmagnifying glass icon at the top to begin a  search of your notes.\n\n\nYou’ll already be in editor mode by default when you open Obsidian and you\ncan edit your notes or write new ones. All markdown syntax is visible in\nthis mode. Press Ctrl + E to switch to preview mode, and the syntax will\ndisappear and the note will appear formatted.\n\n\nIf you type a hashtag before a word, Obsidian will detect it and assign it\nto the note, regardless of where it is in your text.\n\n\n## Get going with Obsidian.md\n\n\n[Obsidan.md](https://obsidian.md), at it's core, is an application that\nhelps manage your markdown files. You can download the application\n\nvia their site and create a \"workspace\" folder when you first start the\napplication. When using the application, all of your notes\n\nwill be created in the folder you choose as your \"workspace\".\n\n\n![Obsidian\napplication](https://about.gitlab.com/images/blogimages/obsidian_md.png){:\n.shadow}\n\n\n### Workspace file structure\n\n\nInside your Obsidian workspace, you can have any number of folders and\nmarkdown files. When you open a folder in Obsidian as your \"workspace\",\n\nObsidian will automatically add a folder `.obsidian`, which contains your\nworkspace configuration such as application styles and plugins.\n\nA basic workspace file structure could look something like this:\n\n\n```\n\n.\n\n├── workspace_folder/\n\n│   └── Other pages/\n\n│   │   └── Another page.md\n\n│   └── .obsidian\n\n│   └── index.md\n\n```\n\n\n`index.md`\n\n```markdown\n\n# Home\n\n\nThis is a basic home page, and a link to another page in my documents.\n\n\nSee [[Another page]] - note that this link uses wikilinks which Obsidian\nuses to help you easily link to other notes in your workspace.\n\n```\n\n\n`Other pages/Another page.md`\n\n```markdown\n\n# Another page\n\n\nThis is another page besides the home page.\n\n```\n\n\n## Generating a static site to host your notes\n\n\nIn order to publish your notes to GitLab Pages, you need to create a static\nsite to show and navigate your notes.\n\nThere are several open source tools that generate static sites from Markdown\ndocuments. After experimenting\n\nwith a few, I found [MkDocs](https://www.mkdocs.org/) to be the easiest and\nmost compatible with Obsidian.\n\n\nIf you would like to use MkDocs locally, you can install it with `pip\ninstall mkdocs`\n\n(Python and [pip as package manager](https://pypi.org/project/pip/) are\nrequired).\n\nThis is not necessary, because in this tutorial we'll utilize GitLab CI\npipelines to install MkDocs and build our site.\n\n\nThere are two small steps you need to make in order to get your existing\nObsidian notes working with MkDocs.\n\n\n### File structure\n\n\nAll files that are not your workspace notes will be created outside of your\nworkspace folder. The following folder structure is\n\nhow this final demo project is going to look.\n\n\n```\n\n.\n\n├── wiki/\n\n│   └── .obsidian\n\n│   └── index.md\n\n├── .gitlab-ci.yml\n\n├── mkdocs.yml\n\n└── requirements.txt\n\n```\n\n - `wiki/` - this is your Obsidian workspace folder\n - `.obsidian` - the application configuration folder Obsidian uses for your workspace. This will not affect the site.\n - `index.md` - MkDocs looks for `index.md` in your workspace folder to use as your site's home page.\n - `.gitlab-ci.yml` - the GitLab CI configuration file used to deploy your site.\n - `mkdocs.yml` - the MkDocs configuration file use to build and customize your site.\n - `requirements.txt` - this file defines the Python package dependencies for MkDocs.\n\n### Basic MkDocs Configuration\n\n\nYou'll need to create a configuration file `mkdocs.yml` for MkDocs to know\nhow you would like your site to look.\n\nHere are the first four lines we need to configure our notes.\n\n\n```yaml\n\nsite_name: My Obsidian Notes\n\nsite_url: https://group-name.gitlab.io/repo-name\n\nsite_dir: public\n\ndocs_dir: ./wiki\n\n```\n\n\n- `site_name` - is what will be used as the main title for the web site.\n\n- `site_url` - is used as the \"canonical URL\" of the site. You will need to\nuse [the default URL provided by GitLab\nPages](https://docs.gitlab.com/ee/user/project/pages/getting_started_part_one.html#gitlab-pages-default-domain-names)\nor your custom domain here.\n\n- `site_dir` - GitLab Pages requires HTML source code to be contained in a\n`public` folder. This setting tells MkDocs to put the generated files in the\n`public` folder.\n\n- `docs_dir` - this is the relative path to your workspace folder. I like to\nname mine `wiki` because it's my personal wikipedia. You can name this\nfolder whatever you want.\n\n\nWe'll come back to this configuration file later to add more custom styles\nto your site.\n\n\n## Configuring GitLab CI\n\n\nWe need to configure a GitLab CI job to install MkDocs and build the web\nsite based on our Obsidian notes. The following\n\n`.gitlab-ci.yml` file has the basic setup for this:\n\n\n```yaml\n\nimage: python:3.8-slim\n\n\npages:\n  stage: deploy\n  script:\n    # Install all of the python packages for mkdocs\n    - pip install -r requirements.txt\n    # Build the site using mkdocs\n    # --strict aborts the build on any warnings\n    # --verbose enables verbose output so that it's easier to see what mkdocs is doing\n    # neither --strict nor --verbose are necessary if you choose not to use them\n    - mkdocs build --strict --verbose\n  artifacts:\n    paths:\n      - public\n  only:\n    - main\n```\n\n\nThis job will only run when a change is made to the default branch (`main`\nin this case).\n\n\n### Python Packages\n\n\nNote the line `pip install -r requirements.txt` in the above\n`.gitlab-ci.yml` file. This line is installing MkDocs and any\n\nadditional plugins you use to customize your site. You'll need to create a\n`requirements.txt` file for this script to work:\n\n\n```text\n\n# Documentation static site generator & deployment tool\n\nmkdocs>=1.1.2\n\n```\n\n\nWe'll come back to this `requirements.txt` file to add a couple more\npackages to customize our site later.\n\n\n## Customizing your site\n\n\nOne of the benefits of using MkDocs is that it has a lot of extensions you\ncan add on to customize your site. You can\n\nchange the theme of the site, which adjusts the colors and layout. You can\nalso add extensions that improve how your\n\nmarkdown notes are displayed and interacted with on the site.\n\n\n### Theme\n\n\nMkDocs includes two built-in themes (`mkdocs` and `readthedocs`), [as\ndocumented on their\nwebsite](https://www.mkdocs.org/user-guide/choosing-your-theme/).\n\nThere are also a lot of [community built\nthemes](https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes) you can search\nthrough and choose to use.\n\nMy current favorite theme is\n[Material](https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes#material-for-mkdocs-).\nYou can install it by adding it our `requirements.txt` and choosing\n\nit as your theme in the `mkdocs.yml` configuration file, or if you are\ninstalling it locally you can install it with `pip install mkdocs-material`.\n\n\n`requirements.txt`\n\n```text\n\n# Material theme\n\nmkdocs-material>=8.1.7\n\n```\n\n\n`mkdocs.yml`\n\n```yaml\n\ntheme:\n  name: material\n  palette:\n    scheme: slate\n```\n\n\nI have chosen the `slate` scheme for the material theme which makes it\ndarker. You can choose more configuration options\n\nbased on [their website\ndocumentation](https://squidfunk.github.io/mkdocs-material/setup/changing-the-colors/).\n\n\n### Extensions\n\n\nMkDocs includes [built-in\nextensions](https://www.mkdocs.org/user-guide/configuration/#markdown_extensions)\nthat you can add to your `mkdocs.yml` configuration file. The\n\n[Material](https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes#material-for-mkdocs-)\ntheme package also comes with many more extensions that we can use. Below\nare some of my favorite\n\nfor working with Obsidian:\n\n\n```yaml\n\n# Extensions\n\nmarkdown_extensions:\n  - footnotes\n  - attr_list\n  - pymdownx.highlight\n  - pymdownx.superfences\n  - pymdownx.details\n  - pymdownx.magiclink\n  - pymdownx.tasklist\n  - pymdownx.emoji\n  - admonition\n  - toc:\n    permalink: true\n```\n\n\n- `footnotes` - adds the ability to define inline footnotes, whech are then\nrendered below all Markdown content of a document. [See documentation\nhere](https://squidfunk.github.io/mkdocs-material/reference/footnotes/).\n\n- `attr_list` - allows you to add HTML attributes and CSS classes to almost\nevery Markdown inline and block-level element with special syntax. [See\ndocumentation\nhere](https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#attribute-lists).\n\n- `pymdownx.highlight` - adds support for syntax highlighting of code\nblocks. [See documentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/highlight/).\n\n- `pymdownx.superfences` - allows for arbitrary nesting of code and content\nblocks inside each other. [See documentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/superfences/).\n\n- `pymdownx.details` - allows for creating collapsible content blocks. [See\ndocumentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/details/).\n\n- `pymdownx.magiclink` - provides a number of useful link related features\nsuch as auto-link HTML and emails. [See documentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/magiclink/).\n\n- `pymdownx.tasklist` - adds support for tasklist syntax. [See documentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/tasklist/).\n\n- `pymdownx.emoji` - adds support for inserting emoji via simple short names\nenclosed within colons (`:short_name:`). [See documentation\nhere](https://facelessuser.github.io/pymdown-extensions/extensions/emoji/).\n\n- `admonition` - allows you to create \"callouts\" in your documentation. [See\ndocumentation\nhere](https://squidfunk.github.io/mkdocs-material/reference/admonitions/).\n\n- `toc:permalink` - adds a table of contents to your page based on your\nmarkdown document, and ensures each link is a permanent link that can be\nreused. [See documentation\nhere](https://python-markdown.github.io/extensions/toc/).\n\n\n### Plugins\n\n\nMkDocs also has a community of plugins that add more features when building\nyour site. MkDocs includes some plugins by default that you can use in the\nconfiguration file, but in order to use community plugins you have to add\nthem to the\n\n`requirements.txt` file to be installed as packages. The following two\nplugins are ones that I've found useful, but you\n\ncan look at [the list of community plugins\nhere](https://github.com/mkdocs/mkdocs/wiki/MkDocs-Plugins):\n\n\n```yaml\n\nplugins:\n  - search\n  - roamlinks\n```\n\n\n- `search` - provides a search bar at the top of your site to easily search\nyour documents. [See documentation\nhere](https://www.mkdocs.org/user-guide/configuration/#search).\n\n- `roamlinks` - adds support for Obsidian's wikilinks feature. [See\ndocumentation here](https://github.com/Jackiexiao/mkdocs-roamlinks-plugin).\n\n\n`requirements.txt`\n\n```text\n\n# Wikilinks support\n\nmkdocs-roamlinks-plugin>=0.1.3\n\n```\n\n\nIf installing locally, you can install roamlinks with `pip install\nmkdocs-roamlinks-plugin`.\n\n\n## Combining it all together\n\n\nAfter all of the above work is done, you should have a file structure that\nlooks like this:\n\n\n```\n\n.\n\n├── wiki/\n\n│   └── .obsidian\n\n│   └── index.md\n\n├── .gitlab-ci.yml\n\n├── mkdocs.yml\n\n└── requirements.txt\n\n```\n\n\nHere are the contents of the three main files that you've been editing:\n\n\n`.gitlab-ci.yml`\n\n```yaml\n\nimage: python:3.8-slim\n\n\npages:\n  stage: deploy\n  script:\n    - pip install -r requirements.txt\n    - mkdocs build --strict --verbose\n  artifacts:\n    paths:\n      - public\n  only:\n    - main\n```\n\n\n`mkdocs.yml`\n\n```yaml\n\nsite_name: My Obsidian Notes\n\nsite_url: https://group-name.gitlab.io/repo-name\n\nsite_dir: public\n\n\ntheme:\n  name: material\n  palette:\n    scheme: slate\n\n# Extensions\n\nmarkdown_extensions:\n  - footnotes\n  - attr_list\n  - pymdownx.highlight\n  - pymdownx.superfences\n  - pymdownx.details\n  - pymdownx.magiclink\n  - pymdownx.tasklist\n  - pymdownx.emoji\n  - admonition\n  - toc:\n    permalink: true\n\nplugins:\n  - search\n  - roamlinks\n```\n\n\n`requirements.txt`\n\n```text\n\n# Documentation static site generator & deployment tool\n\nmkdocs>=1.1.2\n\n\n# Material theme\n\nmkdocs-material>=8.1.7\n\n\n# Wikilinks support\n\nmkdocs-roamlinks-plugin>=0.1.3\n\n```\n\n\nNow that your files are all finished, the last step is to push your changes\nto your GitLab repository and wait for your pipeline\n\nto finish. Once finished, you can go to [your default domain provided by\nGitLab](https://docs.gitlab.com/ee/user/project/pages/getting_started_part_one.html#gitlab-pages-default-domain-names)\nor you can\n\n[configure GitLab Pages to use a custom\ndomain](https://docs.gitlab.com/ee/administration/pages/index.html).\n\n\nHere's a screenshot of the demonstration site created in this tutorial:\n\n\n![Obsidian\napplication](https://about.gitlab.com/images/blogimages/obsidian_mkdocs_site.png){:\n.shadow}\n\n\n## Is the Obsidian note-taking secure?\n\n\nUsers overall believe Obsidian is safe to use. One user said you [maintain\nfull control](https://becomeawritertoday.com/obsidian-review/) over your\nnotes and it provides the ability to encrypt your vault.\n\n\n[This lawyer](https://deskoflawyer.com/secure-note-taking-apps/) maintains\nthat Obsidian is the most-secure note-taking app available. Others claim\nthere are [no security\nthreats](https://thebusinessblocks.com/is-obsidian-one-of-the-most-secure-and-best-notetaking-apps/)\nwith Obsidian and users don’t have to worry about data being lost or\ntransferred to third parties.\n\n\nBecause your files are stored on your own computer, this keeps your data\nsafe and private according to another\n[user](https://www.online-tech-tips.com/computer-tips/how-to-use-obsidian-as-a-personal-wiki-on-your-computer/).\n\n\n### Where to find more information on Obsidian markdown\n\n\nYou can find more information in this [Obsidian markdown\nguide](https://www.markdownguide.org/tools/obsidian/). An Obsidian roadmap\nis available [here](https://trello.com/b/Psqfqp7I/obsidian-roadmap). Of\ncourse, you can also go to the [Obsidan website](https://obsidian.md/).\n\n\nShare your Obsidian.md deployments in the comments.\n",[9],{"slug":3595,"featured":6,"template":700},"publishing-obsidian-notes-with-gitlab-pages","content:en-us:blog:publishing-obsidian-notes-with-gitlab-pages.yml","Publishing Obsidian Notes With Gitlab Pages","en-us/blog/publishing-obsidian-notes-with-gitlab-pages.yml","en-us/blog/publishing-obsidian-notes-with-gitlab-pages",{"_path":3601,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3602,"content":3608,"config":3613,"_id":3615,"_type":13,"title":3616,"_source":15,"_file":3617,"_stem":3618,"_extension":18},"/en-us/blog/quick-start-guide-for-gitlab-workspaces",{"title":3603,"description":3604,"ogTitle":3603,"ogDescription":3604,"noIndex":6,"ogImage":3605,"ogUrl":3606,"ogSiteName":685,"ogType":686,"canonicalUrls":3606,"schema":3607},"Quickstart guide for GitLab Remote Development workspaces","Learn how to create a workspace from your GitLab account and work directly from the remote development environment.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749664219/Blog/Hero%20Images/2023-06-22-quickstart-workspaces-cover-image2.png","https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Quickstart guide for GitLab Remote Development workspaces\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2023-06-26\",\n      }",{"title":3603,"description":3604,"authors":3609,"heroImage":3605,"date":3610,"body":3611,"category":785,"tags":3612},[1096],"2023-06-26","GitLab 16.0 introduced [Remote Development workspaces\n(beta)](https://about.gitlab.com/releases/2023/05/22/gitlab-16-0-released/#remote-development-workspaces-available-in-beta-for-public-projects),\nan exciting addition to the GitLab platform that empowers teams to build and\ndeliver software more efficiently.\n\n\nThis guide provides step-by-step instructions on how to create a workspace\ndirectly from your GitLab account and work directly from the remote\ndevelopment environment. You will work in the Web IDE, a Visual Studio Code\nbrowser version, seamlessly integrated into the workspace. \n\n\nFrom this quick start, you will learn how to create a workspace, use the Web\nIDE Terminal to install dependencies or start your server, and view your\nrunning application. \n\n\nTo learn more about Remote Development in GitLab, we recommend reading this\ninformative blog post, \"[A first look at\nworkspaces](https://about.gitlab.com/blog/introducing-workspaces-beta/),\"\nand the [workspaces docs](https://docs.gitlab.com/ee/user/workspace/).\n\n\nHere are the steps covered in this tutorial:\n\n\n- [Prerequisites](#prerequisites)\n\n- [Locate DevFile at the root of\nrepository](#locate-devfile-at-the-root-of-repository)\n\n- [Create your workspace](#create-your-workspace)\n\n- [Install dependencies and previewing your application in the\nworkspace](#install-dependencies-and-previewing-your-application-in-the-workspace)\n\n- [Make changes to the application and previewing the updated\nversion](#make-changes-to-the-application-and-previewing-the-updated-version)\n\n- [Commit the change](#commit-the-change)\n\n- [Explore the demo](#explore-the-demo)\n\n- [Try out workspaces](#try-out-workspaces)\n\n\n## Prerequisites \n\nPrior to enabling developers to create workspaces, there are a few\nprerequisites such as bring your own Kubernetes cluster, and install and\nconfigure the GitLab agent for Kubernetes on it. Additionally, certain\nconfiguration steps must be completed on the cluster. You can find detailed\ninstructions for all these steps in [our workspaces prequisites\ndocumentation](https://docs.gitlab.com/ee/user/workspace/configuration.html#prerequisites).\nOnce the prerequisites are properly configured, developers who hold\nDeveloper role or above within the root group will gain the ability to\ncreate workspaces.\n\n\n## Locate DevFile at the root of repository\n\nA [devfile](https://devfile.io/docs/2.2.0/devfile-ecosystem) is a\ndeclarative configuration file, in YAML syntax, used to define and describe\nthe development environment for a software project. It provides a\nstandardized way to specify the necessary tools, languages, runtimes, and\nother components required for developing an application.\n\n\nTo initiate a workspace, it is necessary to have a devfile located at the\nroot of the repository. In this blog post, we will utilize a project that\ncontains a devfile, accessible\n[here](https://gitlab.com/gitlab-da/use-cases/remote-development/example-nodejs-express-app/-/raw/main/.devfile.yaml). \n\n\n```yaml\n\nschemaVersion: 2.2.0\n\ncomponents:\n  - name: tooling-container\n    attributes:\n      gl/inject-editor: true\n    container:\n      # NOTE: THIS IMAGE EXISTS ONLY FOR DEMO PURPOSES AND WILL NOT BE MAINTAINED\n      image: registry.gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/debian-bullseye-ruby-3.2-node-18.12:rubygems-3.4-git-2.33-lfs-2.9-yarn-1.22-graphicsmagick-1.3.36-gitlab-workspaces\n      memoryRequest: 1024M\n      memoryLimit: 2048M\n      cpuRequest: 500m\n      cpuLimit: 1000m\n      endpoints:\n      - name: http-3000\n        targetPort: 3000\n```\n\nFor more information, see the [GitLab\ndocumentation](https://docs.gitlab.com/ee/user/workspace/#devfile) and\n[devfile documentation](https://devfile.io/docs/2.2.0/devfile-schema).\n\n\n## Create your workspace \n\n1. Make sure you have a [Developer role or\nabove](https://docs.gitlab.com/ee/user/permissions.html) in the root group,\nand the above prerequisites configured properly.\n\n2. Fork [this\nproject](https://gitlab.com/gitlab-da/use-cases/remote-development/example-nodejs-express-app)\nto the GitLab group for which you have a Developer role or above. \n\n3. Switch contexts and select `Your work`.\n\n![Your\nwork](https://about.gitlab.com/images/blogimages/2023-07-10-your-work.png){:\n.shadow}\n\n4. Select `Workspaces`.\n\n5. Select `New workspace`.\n\n6. Select the project you forked or another project that has a\n`.devfile.yaml` file at the root of the repository. \n\n7. Select the [cluster\nagent](https://docs.gitlab.com/ee/user/workspace/#prerequisites) owned by\nthe group the project belongs to.\n\n8. In `Time before automatic termination`, enter the number of hours until\nthe workspace automatically terminates. This timeout is a safety measure to\nprevent a workspace from consuming excessive resources or running\nindefinitely. \n\n9. Select `Create workspace`. \n\n\n![create\nws](https://about.gitlab.com/images/blogimages/create_workspace.png){:\n.shadow}\n\n\nThe workspace will be deployed to the cluster and might take a few minutes\nto start. To access the workspace, under Preview, select the workspace link.\n\n\n![ws list](https://about.gitlab.com/images/blogimages/workspaces_list.png){:\n.shadow}\n\n\n## Install dependencies and previewing your application in the workspace\n\nAfter creating your workspace, the [Web IDE using VS\nCode](https://docs.gitlab.com/ee/user/workspace/#web-ide) is injected into\nit, and the repository is cloned to the image. Consequently, you gain\nimmediate access to your code and can commence working on it right away.\n\n\nYou can now open the terminal, install any missing dependencies, and start\nthe application.\n\n\n![Terminal](https://about.gitlab.com/images/blogimages/ws-terminal.png){:\n.shadow}\n\n\n1. To open the terminal, from the left menu, select `Terminal`, `New\nTerminal`. \n\n2. Type `npm install` to install the dependencies listed in the\n[package.json](https://gitlab.com/gitlab-da/use-cases/remote-development/example-nodejs-express-app/-/blob/main/package.json)\nfile.\n\n3. Type `npm start` to start the application.\n\n\nThe log will indicate that the application has started on port 3000.\n\n\n![log](https://about.gitlab.com/images/blogimages/server_log.png){: .shadow}\n\n\nYou can now access your application by opening the browser and using the\nworkspace URL. Change the number before ‘workspace’ in the URL to the port\nnumber on which your application is listening (e.g., 3000). For example, if\nyour workspace URL is\n`https://\u003Cprefix>-workspace-73241-25728545-rqvpjm.workspaces.gitlab.dev`,\nand your application is running on port 3000, update `\u003Cprefix>` to 3000 to\naccess your application.\n\n\n## Make changes to the application and previewing the updated version\n\nIn the Web IDE, navigate to the `server.js` file, modify the text in line\n9. \n\n\nAfterward, refresh the browser where your application is opened to see the\napplied changes. \n\n\n## Commit the change \n\n1. In the Web IDE click on the merge icon in the activity bar.\n\n2. Click the line with the `server.js` to view your change side by side.\n\n3. To stage your change, click the plus icon next to `server.js`.\n\n4. Type a commit message describing your change.\n\n5. Click Commit. \n\n6. Click Sync changes to push the commit to the GitLab server.\n\n  ![commit](https://about.gitlab.com/images/blogimages/commit-stage.png){: .shadow}\n\n## Explore the demo \n\nExplore further with this [click-through demo of\nworkspaces](https://go.gitlab.com/qtu66q).\n\n\n## Try out workspaces\n\nRemote Development workspaces offer a convenient and efficient way to work\non projects without the need for local development setups. They provide a\nstreamlined workflow and enable developers to focus on writing code rather\nthan dealing with complex environment setups.\n\n\nBy adopting workspaces, developers can collaborate effectively, improve\nproductivity, and simplify the development process. \n\n\nGive workspaces a try and revolutionize your remote development experience\ntoday!\n\n\nCover image by \u003Ca\nhref=\"https://unsplash.com/@pankajpatel?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">Pankaj\nPatel\u003C/a> on \u003Ca\nhref=\"https://unsplash.com/photos/_SgRNwAVNKw?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText\">Unsplash\u003C/a>\n\n{: .note}\n",[9,789,108],{"slug":3614,"featured":6,"template":700},"quick-start-guide-for-gitlab-workspaces","content:en-us:blog:quick-start-guide-for-gitlab-workspaces.yml","Quick Start Guide For Gitlab Workspaces","en-us/blog/quick-start-guide-for-gitlab-workspaces.yml","en-us/blog/quick-start-guide-for-gitlab-workspaces",{"_path":3620,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3621,"content":3627,"config":3632,"_id":3634,"_type":13,"title":3635,"_source":15,"_file":3636,"_stem":3637,"_extension":18},"/en-us/blog/quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai",{"title":3622,"description":3623,"ogTitle":3622,"ogDescription":3623,"noIndex":6,"ogImage":3624,"ogUrl":3625,"ogSiteName":685,"ogType":686,"canonicalUrls":3625,"schema":3626},"Quick vulnerability remediation with GitLab Advanced SAST + Duo AI ","Shorten your mean time to remediation by pairing Advanced SAST and artificial intelligence. This detailed demo shows you how.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098458/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945_24mPf16vAPHORs3d9y62q_1750098458538.png","https://about.gitlab.com/blog/quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Quick vulnerability remediation with GitLab Advanced SAST + Duo AI \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2024-10-22\",\n      }",{"title":3622,"description":3623,"authors":3628,"heroImage":3624,"date":3629,"body":3630,"category":718,"tags":3631},[1444],"2024-10-22","With GitLab 17.4, we’ve made [GitLab Advanced SAST generally\navailable](https://about.gitlab.com/blog/gitlab-advanced-sast-is-now-generally-available/).\n[GitLab Advanced\nSAST](https://docs.gitlab.com/ee/user/application_security/sast/gitlab_advanced_sast.html)\nis a static application security testing scanner designed to discover\nvulnerabilities by performing cross-function and cross-file taint analysis.\nBy following the paths user inputs take, the analyzer identifies potential\npoints where untrusted data can influence the execution of your application\nin unsafe ways, ensuring the vulnerabilities are detected even when they\nspan multiple functions and files.\n\n\nGitLab Advanced SAST can be used together with [GitLab Duo Vulnerability\nExplanation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#explaining-a-vulnerability)\nin order to reduce the mean time to remediation (MTTR). GitLab Duo can\nprovide practical, AI-powered examples of how threat actors can exploit\nvulnerabilities and offer light-weight remediation guidance, which can be\nused with cross-file analysis to enhance application security (AppSec)\nefficiency.\n\n\nThis tutorial will show you how to:\n\n* enable GitLab Advanced SAST\n\n* read results from the scanner\n\n* review the code flow of a vulnerability\n\n* use GitLab AI to quickly remediate the vulnerability\n\n\n## Enable GitLab Advanced SAST\n\n\nFollow the instructions below to enable GitLab Advanced SAST. You can also\nview this video to get started:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/xDa1MHOcyn8?si=5SYuKgP-BdBryqcU\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Run GitLab Advanced SAST on each code commit\n\n\nBefore using Advanced SAST, the following prerequisites must be met:\n\n\n- GitLab Ultimate Subscription ([free\ntrial](https://gitlab.com/-/trials/new?glm_content=default-saas-trial&glm_source=about.gitlab.com%2F))\n\n- GitLab SaaS or GitLab Self-managed (running Version 17.4)\n\n\nTo enable the GitLab Advanced SAST scanner:\n\n\n- On the left sidebar, select **Search** or **Go to** and find your project.\n\n- Add or edit the `.gitlab-ci.yml` to include the following:\n    - Test stage\n    - `Jobs/SAST.gitlab-ci.yml` template\n    - `GITLAB_ADVANCED_SAST_ENABLED` variable set to true\n- Apply the change.\n\n\nYour newly merged `.gitlab-ci.yml` should contain the following:\n\n\n```yaml\n\nstages:\n  - test\n\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml\n\nvariables:\n  GITLAB_ADVANCED_SAST_ENABLED: 'true'\n```\n\n\nThis will now run the `gitlab-advances-sast` job within the test stage of\nyour application along with all the other jobs you have defined. Advanced\nSAST will replace the semgrep SAST scanner for the [supported programming\nlanguages](https://docs.gitlab.com/ee/user/application_security/sast/gitlab_advanced_sast.html#supported-languages).\n\n\n![Running `gitlab-advances-sast` job within the test stage of your\napplication](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098466/Blog/Content%20Images/Blog/Content%20Images/1_aHR0cHM6_1750098466629.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Advanced SAST job in pipeline\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\n**Note:** You can fully configure the job as you would any job in GitLab.\nFor more information, see the [CI/CD YAML syntax\ndocumentation](https://docs.gitlab.com/ee/ci/yaml/).\n\n\n## Remediate vulnerabilities in merge request (pre-production)\n\n\nJust like our previous SAST scanner, Advanced SAST allows you to scan source\ncode in the diff of a feature branch. This allows us to address any incoming\nvulnerabilities before they make it into production. Here we can see the\nscanner results for the diff within a merge request:\n\n\n![Advanced SAST scanner results for the diff within a merge\nrequest](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/2_aHR0cHM6_1750098466630.png)\n\n\nWhen selecting a newly detected vulnerability, we get the following details\nto assist with remediation:\n\n\n- **Status:** The status of the vulnerability (Needs triage, Confirmed,\nDismissed, Resolved)\n\n- **Description:** Detailed information on the detected vulnerability\n\n- **Detection time:** Time vulnerability was detected\n\n- **Location:** Line of code where vulnerability is detected\n\n- **Severity:** Severity of vulnerability from CVE database\n\n- **Training:** Gamified training from our partners\n\n- **Solutions:** Information on how to remediate or resolve a vulnerability\n\n- **Identifiers:** Relevant links showcasing detailed description,\nexploitation, and remediation\n\n\n![Merge request with vulnerability\ninsights](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/MR_with_vulnerability_insights_aHR0cHM6_1750098466632.png)\n\n\n\u003Ccenter>\u003Ci>Merge request with vulnerability insights\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br> \n\nVulnerabilities detected within an MR are actionable, meaning they can be\ndismissed or an issue can be created and populated with relevant\nvulnerability information.\n\n\nDismissing an issue saves AppSec teams time, because they can see relevant\ndeveloper information when reviewing an MR. Creating a confidential issue\nallows developers and AppSec teams to further collaborate on resolving a\nvulnerability where a fix is not straightforward. Confidential issues have\nlimited permissions and can be used with confidential merge requests to\nprevent possible malicious actors from exploiting.\n\n\nTo further support separation of duties and prevent vulnerable code from\nmaking it into production, you can require approval from certain people (for\nexample, the security team) in order to merge vulnerable code.\n\n\n![GitLab security policies in\naction](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/security_policies_in_action_aHR0cHM6_1750098466634.png)\n\n\n\u003Ccenter>\u003Ci>Security policies in action\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\n**Note:** Learn more about Security Policies and how to implement them in\nthe [Security Policy\ndocumentation](https://docs.gitlab.com/ee/user/application_security/policies/).\n\n\n## Manage vulnerabilities in production\n\n\nWhile preventing vulnerabilities from making it into production is crucial\nfor application security, it is equally as important to manage\nvulnerabilities in production. When security scanners are run on a default\nor production-level branch, a [vulnerability\nreport](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/)\nwill be populated with the latest vulnerability data which can be used to\ntriage and manage vulnerabilities.\n\n\n![GitLab Vulnerability Report sorted by Advanced\nSAST](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/5_aHR0cHM6_1750098466636.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Vulnerability Report sorted by Advanced SAST\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nWhen selecting a vulnerability you get similar vulnerability details as seen\nin a merge request, making for a single source of truth for developers and\nAppSec teams.\n\n\n![Vulnerability page with vulnerability\ninsights](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/Vuln_page_with_vulnerability_insights_aHR0cHM6_1750098466637.png)\n\n\n\u003Ccenter>\u003Ci>Vulnerability page with vulnerability insights\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\nAppSec teams can triage a vulnerability by changing its status and adding\nrelevant details on the status change. Issues can be created to track the\nprogress of a fix. From here, a developer can be assigned.\n\n\n## Examine vulnerable code flow\n\n\nFor vulnerabilities detected with Advanced SAST, we can see a \"Code flow\"\ntab on the Vulnerability page.\n\n\n![Advanced SAST - image\n7](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/7_aHR0cHM6_1750098466638.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Advanced SAST code flow\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nIn this example, you can see that a vulnerability is traced across multiple\nfunctions, giving deeper insight into the best practices we should put in\nplace to not only resolve the vulnerability, but prevent similar\nvulnerabilities in the future.\n\n\n## Use GitLab Duo Vulnerability Explanation\n\n\nGitLab Duo can help you mitigate or remediate a vulnerability by using a\nlarge language model to:\n\n\n- Summarize the vulnerability\n\n- Help developers and security analysts understand the vulnerability\n\n- Show how the vulnerability can be exploited\n\n- Provide a suggested remediation or mitigation\n\n\nTo use Vulnerability Explanation, the following is required:\n\n\n- GitLab Ultimate subscription\n\n- GitLab Duo Enterprise seat\n\n- GitLab Duo must be enabled for your group or instance\n\n\nFrom the vulnerability report, you can select a SAST vulnerability and go to\nits Vulnerability page. From the Vulnerability page, you can do any of the\nfollowing to explain the vulnerability:\n\n\n- Select the text below the vulnerability description\n\n- You can use AI by asking GitLab Duo Chat to explain this vulnerability and\noffer a suggested fix.\n\n- In the upper right, from the \"Resolve with merge request\" dropdown list,\nselect **Explain Vulnerability**, then select **Explain vulnerability**.\n\n- Open GitLab Duo Chat and use the explain a vulnerability command:\n`/vulnerability_explain`.\n\n\nThen the vulnerable code will be processed by Anthropic’s Claude 3 Haiku\nmodel and provide the following data:\n\n\n![GitLab Duo Vulnerability\nExplanation](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098467/Blog/Content%20Images/Blog/Content%20Images/vuln_explain_2_aHR0cHM6_1750098466640.png)\n\n\n## Putting it all together\n\n\nNow, let's put it all together with a concrete example. I will use the\n[OWASP Juice Shop](https://owasp.org/www-project-juice-shop/) as my demo\napplication and run GitLab Advanced SAST to detect a vulnerability in\nproduction. Then I will use the vulnerability code flow and GitLab Duo to\ninvestigate vulnerability exploitation, and remediation. You can [follow\nalong with this\ndemo](https://gitlab.com/gitlab-da/tutorials/security-and-governance/owasp/juice-shop)\nand see this workflow in action by watching:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/H1S43oM44k0?si=2LYorTjByOHbCAko\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nThe detection and remediation workflow is as follows:\n\n\n- Enable GitLab Advanced SAST and run it on the project’s default branch.\n\n- Open the Vulnerability Report and sort by **Tool:GitLab Advanced SAST**.\n\n- Select the **Improper neutralization of special elements in data query\nlogic** vulnerability found in `Basket.ts`.\n\n- Use the vulnerability code flow to understand the vulnerable paths.\n\n- Run **Explain this vulnerability** to see exploit information.\n\n- Run the application locally to attempt exploitation.\n\n- Change vulnerability status to \"Confirmed\" and provide relevant info.\n\n- Determine remediation path using all relevant data:\n    - Vulnerability page insights, Code Flow, Vulnerability Explanation results\n- Create a new branch and apply remediation.\n\n- Run the remediated application locally and try to exploit again.\n\n- Create a merge request with the fix.\n\n- Code change will be tested using CI to assure we don’t break the\napplication.\n\n- Validate and merge MR.\n\n- Test exploit in deployed environment.\n\n- Change vulnerability status to \"Resolved\" on the Vulnerability page.\n\n\n**Note:** There are many ways to triage and remediate vulnerabilities, make\nsure to follow best practices set by your organization.\n\n\n# Useful links\n\n\nTo learn more about GitLab and how you can get started with enhancing your\norganization’s application security posture, check out the following\nresources.\n\n\n* [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/) \n\n* [GitLab Duo](https://about.gitlab.com/gitlab-duo/)  \n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab Software Supply Chain Security\nSolutions](https://about.gitlab.com/solutions/supply-chain/)  \n\n* [GitLab Continuous Software\nCompliance](https://about.gitlab.com/solutions/continuous-software-compliance/)  \n\n* [JuiceShop Demo\nApplication](https://gitlab.com/gitlab-da/tutorials/security-and-governance/owasp/juice-shop)  \n\n* [GitLab AppSec\ndocumentation](https://docs.gitlab.com/ee/user/application_security/)  \n\n* [Advanced SAST \ndocumentation](https://docs.gitlab.com/ee/user/application_security/sast/gitlab_advanced_sast.html)  \n\n* [Explain this Vulnerability\ndocumentation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#explaining-a-vulnerability)  \n\n* [Code Flow\ndocumentation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#vulnerability-code-flow)  \n\n* [Security Policy\ndocumentation](https://docs.gitlab.com/ee/user/application_security/policies/) \n\n* [OWASP Juice Shop\ndocumentation](https://owasp.org/www-project-juice-shop/)\n",[720,830,9,696,495],{"slug":3633,"featured":90,"template":700},"quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai","content:en-us:blog:quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai.yml","Quick Vulnerability Remediation With Gitlab Advanced Sast Duo Ai","en-us/blog/quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai.yml","en-us/blog/quick-vulnerability-remediation-with-gitlab-advanced-sast-duo-ai",{"_path":3639,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3640,"content":3646,"config":3650,"_id":3652,"_type":13,"title":3653,"_source":15,"_file":3654,"_stem":3655,"_extension":18},"/en-us/blog/quickly-resolve-broken-ci-cd-pipelines-with-ai",{"title":3641,"description":3642,"ogTitle":3641,"ogDescription":3642,"noIndex":6,"ogImage":3643,"ogUrl":3644,"ogSiteName":685,"ogType":686,"canonicalUrls":3644,"schema":3645},"Quickly resolve broken CI/CD pipelines with AI","When your CI/CD pipeline fails, it leads to delays, decreased productivity, and stress. AI-powered Root Cause Analysis makes problem-solving faster and smarter.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097355/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2811%29_78Dav6FR9EGjhebHWuBVan_1750097355230.png","https://about.gitlab.com/blog/quickly-resolve-broken-ci-cd-pipelines-with-ai","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Quickly resolve broken CI/CD pipelines with AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2024-12-03\",\n      }",{"title":3641,"description":3642,"authors":3647,"heroImage":3643,"date":2758,"body":3648,"category":718,"tags":3649},[1096],"CI/CD pipelines are the backbone of efficiency in software development. They help teams test, build, and deploy code quickly. But when these pipelines break, everything slows down — deadlines get missed, and developers are left frustrated as they work to fix things and keep projects on track.\n\n![CI/CD pipeline with multiple failed jobs](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097362/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097362772.png)\n\n\u003Ccenter>\u003Ci>CI/CD pipeline with multiple failed jobs\u003C/i>\u003C/center>\u003Cbr>\u003C/br>\n\n**So, why do pipelines break in the first place?** Let’s break it down.\n\n## Reasons for pipeline failures\n\nA pipeline failure occurs when the automated workflow in your [CI/CD pipeline](https://about.gitlab.com/topics/ci-cd/cicd-pipeline/) — a series of steps that can include building, testing, and deploying code — does not execute as expected and ends with an error message. This failure can prevent code from being properly built, tested, or deployed, causing delays in software delivery and requiring troubleshooting to resolve. \n\nPipeline failures can happen for a variety of reasons. Some common causes include:\n- Syntax errors: A small mistake in the code, like a missing semicolon or incorrect variable name, can cause the pipeline to fail.\n- Failed tests: Unit or integration tests might fail due to broken code, incorrect configurations, or mismatched dependencies.\n- Misconfigurations: Incorrect pipeline settings or environment configurations can lead to failed builds or deployments.\n\nThere are also more complex issues that add to the challenge:\n- Infrastructure-as-Code ([IaC](https://about.gitlab.com/topics/gitops/infrastructure-as-code/)) issues: Problems in provisioning cloud infrastructure, such as errors in Terraform scripts or CloudFormation templates, can prevent a successful deployment.\n- Kubernetes and GitOps challenges: Misconfigurations in [Kubernetes clusters](https://about.gitlab.com/blog/kubernetes-the-container-orchestration-solution/) or issues with [GitOps](https://about.gitlab.com/topics/gitops/) workflows (e.g., syncing Kubernetes states with Git repositories) can cause pipeline failures that are difficult to diagnose.\n- Long, messy stack traces: When an error occurs deep in the system, stack traces can become long and hard to decipher, especially when they span multiple components or services.\n\nThese challenges make troubleshooting more difficult and time-consuming, as finding the root cause often involves sifting through complex logs, reviewing configuration files, and testing different solutions.\n\n## The real impact of failed pipelines\n\nWhen a pipeline fails, it doesn’t just delay your deployment — it brings stress and frustration. Developers are forced to pause their work and dive into troubleshooting, which often leads to a chain reaction of disruptions. This makes it harder to meet deadlines and increases the pressure on the entire team. But why is manual troubleshooting so stressful?\n\n### Manual troubleshooting \n\nThe time it takes to fix a broken pipeline varies. It depends on things like:\n- How well the developer knows the project\n- How experienced they are with similar issues\n- Their overall problem-solving skills\n\nManually digging through logs to figure out what went wrong is a tough and tedious process. Logs can come from all over the place, including application errors and system messages, and they’re often messy and hard to interpret. And on top of that, fixing the pipeline usually requires a lot of jumping back and forth between tasks, adding more time to the process.\n\nThis is where [GitLab Duo](https://about.gitlab.com/gitlab-duo/) comes in. GitLab Duo can sift through all that messy data and spot issues much faster, simplifying the process so you don’t need to be an expert to figure out what went wrong. With AI, fixing your pipelines becomes faster, easier, and much less stressful.\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176104/Blog/zxvvu7p9vc3qpmwl32ya.png\" alt=\"broken pipeline\">\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176108/Blog/bpx6dqilfhltzboyp8k8.png\" alt=\"fix suggestions for broken pipelines\">\n\n## GitLab Duo Root Cause Analysis with generative AI\n\nWhen your CI/CD pipeline breaks, you don’t have to spend hours manually troubleshooting. Enter [GitLab Duo’s Root Cause Analysis (RCA)](https://docs.gitlab.com/ee/user/gitlab_duo/#root-cause-analysis). This AI-powered tool quickly identifies the exact cause of the failure and suggests fixes — right within the DevSecOps platform. No matter how long or complicated your stack traces are, RCA analyzes all the data, breaks it down, and gives you clear, actionable insights.\n\n**It tells you exactly what caused the error, provides steps to fix it, and even pinpoints the specific files and lines of code that need attention.** And, to make it even easier, it suggests code fixes to get everything back on track. This makes troubleshooting a lot faster and more straightforward.\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176111/Blog/nmagby9hoksskogve53m.png\" alt=\"root cause of failure\">\n\n\u003Cimg src=\"https://res.cloudinary.com/about-gitlab-com/image/upload/v1752176115/Blog/dndis1cedwbmbnj33q3v.png\" alt=\"example fix\">\n\n## Keep the conversation going with follow-up questions\n\nWith GitLab Duo RCA, you don’t just get answers — you can ask follow-up questions to dig deeper. Want to explore alternative solutions? No problem. You can add [more context](https://docs.gitlab.com/ee/user/gitlab_duo_chat/index.html#the-context-chat-is-aware-of) by referencing other files, issues, or epics in your repo. For example, you could open your `.gitlab-ci.yml` file in the IDE and ask the chat, “Based on this file, and the analyzed CI/CD pipeline, how would you propose to optimize the pipeline?” \n\n## Privacy first – everything stays in GitLab\nOne of the key benefits of GitLab Duo RCA is that it works right out of the box within GitLab. You won’t have to switch tools or go hunting for external help. Plus, your [logs and sensitive data stay secure](https://about.gitlab.com/privacy/) - there’s no need to send them off to external AI solutions. RCA is seamlessly integrated within GitLab, offering valuable insights without ever compromising privacy.\n\n![broken pipelines - image 6](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097363/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097362773.png)\n\n## Get started today\n\nWant to see how AI can supercharge your development process, making it smoother and faster? Dive into our GitLab Duo Enterprise product tour below and discover how GitLab Duo’s AI-powered insights can transform every stage of your development journey — from planning and coding to troubleshooting and deployment. Click the image below to start the tour!\n\n[![GitLab Duo Enterprise tour](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097363/Blog/Content%20Images/Blog/Content%20Images/Screenshot_2024-12-02_at_12.41.10_PM_aHR0cHM6_1750097362774.png)](https://gitlab.navattic.com/duo-enterprise)\n\n> [Start a free trial of GitLab Duo today!](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/)",[720,495,9,696],{"slug":3651,"featured":6,"template":700},"quickly-resolve-broken-ci-cd-pipelines-with-ai","content:en-us:blog:quickly-resolve-broken-ci-cd-pipelines-with-ai.yml","Quickly Resolve Broken Ci Cd Pipelines With Ai","en-us/blog/quickly-resolve-broken-ci-cd-pipelines-with-ai.yml","en-us/blog/quickly-resolve-broken-ci-cd-pipelines-with-ai",{"_path":3657,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3658,"content":3664,"config":3670,"_id":3672,"_type":13,"title":3673,"_source":15,"_file":3674,"_stem":3675,"_extension":18},"/en-us/blog/rebase-in-real-life",{"title":3659,"description":3660,"ogTitle":3659,"ogDescription":3660,"noIndex":6,"ogImage":3661,"ogUrl":3662,"ogSiteName":685,"ogType":686,"canonicalUrls":3662,"schema":3663},"How to use Git rebase in real life","From fixup to autosquash here are real world ways to leverage Git rebase.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682486/Blog/Hero%20Images/rebase-in-real-life.jpg","https://about.gitlab.com/blog/rebase-in-real-life","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use Git rebase in real life\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Toon Claes\"}],\n        \"datePublished\": \"2022-11-08\",\n      }",{"title":3659,"description":3660,"authors":3665,"heroImage":3661,"date":3667,"body":3668,"category":785,"tags":3669},[3666],"Toon Claes","2022-11-08","My colleague [Chris](/company/team/#chriscool) recently wrote about [how to\ntake advantage of Git\n\nrebase](/blog/take-advantage-of-git-rebase/). In this post we'll\n\nexplain how you can take these techniques, and apply them to daily developer\nlife.\n\n\n## Fixup\n\n\nImagine you have created a merge request, and there are some pipeline\nfailures\n\nand some comments from reviews, and suddenly your [commit\nhistory](/blog/keeping-git-commit-history-clean/) looks something\n\nlike this:\n\n\n```shell\n\n$ git log --oneline\n\n\n8f8ef5af (HEAD -> my-change) More CI fixes\n\ne4fb7935 Apply suggestion from reviewer\n\nc1a1bec6 Apply suggestion from reviewer\n\n673222be Make linter happy\n\na0c30577 Fix CI failure for X\n\n5ff160db Implement feature Y\n\nf68080e3 Implement feature X\n\n3cdbc201 (origin/main, origin/HEAD, main) Merge branch 'other-change' into\n'main'\n\n...\n\n```\n\n\nIn this example there are 2 commits implementing feature X and Y, followed\nby a\n\nhandful of commits that aren't useful on their own. We used the fixup\nfeature of\n\nGit rebase to get rid of them.\n\n\n### Finding the commit\n\n\nThe idea of this technique is to integrate the changes of these follow-up\n\ncommits into the commits that introduced each feature. This means for each\n\nfollow-up commit we need to determine which commit they belong to.\n\n\nBased on the filename you may already know which commits belong together,\nbut if\n\nyou don't you can use git-blame to find the commit.\n\n\n```shell\n\ngit blame \u003Crevision> -L\u003Cstart>,\u003Cend> \u003Cfilename>\n\n```\n\n\nWith the option `-L` we'll specify a range of a line numbers we're\ninterested in.\n\nHere `\u003Cend>` cannot be omitted, but it can be the same as `\u003Cstart>`. You can\n\nomit `\u003Crevision>`, but you probably shouldn't because you want to skip over\nthe\n\ncommits you want to rebase away. Your command will look something like this:\n\n\n```shell\n\n$ git blame 5ff160db -L22,22 app/model/user.rb\n\n\nf68080e3 22) scope :admins, -> { where(admin: true) }\n\n```\n\n\nThis tells us line `22` was touched by `f68080e3 Implement feature X`.\n\n\nNow repeat this step until you know the commit for each of the commits you\nwant\n\nto rebase out.\n\n\n### Interactive rebase\n\n\nThe next step is to start the interactive rebase:\n\n\n```shell\n\n$ git rebase -i main\n\n```\n\n\nHere you're presented with the list of instructions in your `$EDITOR`:\n\n\n``` text\n\npick 8f8ef5af More CI fixes\n\npick e4fb7935 Apply suggestion from reviewer\n\npick c1a1bec6 Apply suggestion from reviewer\n\npick 673222be Make linter happy\n\npick a0c30577 Fix CI failure for X\n\npick 5ff160db Implement feature Y\n\npick f68080e3 Implement feature X\n\n```\n\n\nNow you'll need to change these instructions to something like this:\n\n\n```text\n\nfixup 8f8ef5af More CI fixes\n\nfixup e4fb7935 Apply suggestion from reviewer\n\nfixup 673222be Make linter happy\n\npick 5ff160db Implement feature Y\n\nfixup c1a1bec6 Apply suggestion from reviewer\n\nfixup a0c30577 Fix CI failure for X\n\npick f68080e3 Implement feature X\n\n```\n\n\nAs you can see I've reordered the commits, and I've changed some occurrences\nof\n\n`pick` to `fixup`.\n\n\nThe Git rebase will process this list bottom-to-top. It takes each line with\n\n`pick` and uses its commit message. On each line starting with `fixup` it\n\nintegrates the changes into the commit below. When you've saved this file\nand\n\nclosed your `$EDITOR`, the Git history will look something like this:\n\n\n```shell\n\n$ git log --oneline\n\n\ne880c726 (HEAD -> my-change) Implement feature Y\n\ne088ea06 Implement feature X\n\n3cdbc201 (origin/main, origin/HEAD, main) Merge branch 'other-change' into\n'main'\n\n...\n\n```\n\n\n## Autosquash\n\n\nUsing autosquash can be an alternative technique to the above. First we'll\n\nuncommit all the commits we want to get rid of.\n\n\n```shell\n\ngit checkout f68080e3\n\n```\n\n\nNow all changes only exist in your working tree, and are gone from the\ncommit\n\nhistory. You can use `git add` or `git add -p` to stage all changes related\nto\n\n`e088ea06 Implement feature X`. Instead of running `git commit` or `git\ncommit -m`\n\nwe'll use the `--fixup` option:\n\n\n```shell\n\n$ git commit --fixup e088ea06\n\n```\n\n\nNow the history will look something like:\n\n\n```shell\n\n$ git log --oneline\n\n\ne744646b (HEAD -> my-change) fixup! Implement feature X\n\n5ff160db Implement feature Y\n\nf68080e3 Implement feature X\n\n3cdbc201 (origin/main, origin/HEAD, main) Merge branch 'other-change' into\n'main'\n\n...\n\n```\n\n\nAll remaining changes should now belong to `5ff160db Implement feature Y` so\nwe\n\ncan run:\n\n\n```shell\n\n$ git add .\n\n\n$ git commit --fixup 5ff160db\n\n\n$ git log --oneline\n\n\n18c0fff9 (HEAD -> my-change) fixup! Implement feature Y\n\ne744646b fixup! Implement feature X\n\n5ff160db Implement feature Y\n\nf68080e3 Implement feature X\n\n3cdbc201 (origin/main, origin/HEAD, main) Merge branch 'other-change' into\n'main'\n\n...\n\n```\n\n\nYou can now review the `fixup!` commits and if you're happy with it, run:\n\n\n```shell\n\n$ git rebase -i --autosquash main\n\n```\n\n\nYou see we provide the extra option `--autosquash`. This option will look\nfor\n\n`fixup!` commits and automatically reorder those and set their instruction\nto\n\n`fixup`. Normally there's nothing for you to be done now, and you can just\nclose\n\nthe instruction list in your editor. If you type `git log` now you'll see\nthe\n\n`fixup!` commits are gone.\n\n\n## Alternatives\n\n\nFinally, there are some tools that allow you to _absorb_ commits more\neasily, for\n\nexample:\n\n\n* [lib.rs/crates/git-absorb](https://lib.rs/crates/git-absorb)\n\n* [github.com/MrFlynn/git-absorb](https://github.com/MrFlynn/git-absorb)\n\n* [gitlab.com/bertoldia/git-absorb](https://gitlab.com/bertoldia/git-absorb)\n\n* [github.com/tummychow/git-absorb](https://github.com/tummychow/git-absorb)\n\n*\n[github.com/torbiak/git-autofixup](https://github.com/torbiak/git-autofixup)\n\n\n[Cover image](https://unsplash.com/photos/qAShc5SV83M) by [Yung\nChang](https://unsplash.com/@yungnoma) on [Unsplash](https://unsplash.com/).\n\n{: .note}\n",[742,956,9],{"slug":3671,"featured":6,"template":700},"rebase-in-real-life","content:en-us:blog:rebase-in-real-life.yml","Rebase In Real Life","en-us/blog/rebase-in-real-life.yml","en-us/blog/rebase-in-real-life",{"_path":3677,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3678,"content":3683,"config":3687,"_id":3689,"_type":13,"title":3690,"_source":15,"_file":3691,"_stem":3692,"_extension":18},"/en-us/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo",{"title":3679,"description":3680,"ogTitle":3679,"ogDescription":3680,"noIndex":6,"ogImage":947,"ogUrl":3681,"ogSiteName":685,"ogType":686,"canonicalUrls":3681,"schema":3682},"Refactor code into modern languages with AI-powered GitLab Duo ","This detailed tutorial helps developers use AI to modernize code by switching to a new programming language and gain knowledge about new features in the same language.","https://about.gitlab.com/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Refactor code into modern languages with AI-powered GitLab Duo \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2024-08-26\",\n      }",{"title":3679,"description":3680,"authors":3684,"heroImage":947,"date":2538,"body":3685,"category":718,"tags":3686},[1646],"Whether you are tasked with modernizing the code base or framework by\nswitching to a new programming language, or you need knowledge about new\nlanguage features in the same language, AI-powered [GitLab\nDuo](https://about.gitlab.com/gitlab-duo/) can help. Learn how to approach\ncode refactoring challenges with best practices using examples from the past\n20 years of my coding career. \n\n\nThe prompts and examples in this article are shown in different IDEs: VS\nCode and JetBrains IDEs (IntelliJ IDEA, PyCharm, and CLion) with the [GitLab\nDuo\nextensions/plugins](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html)\ninstalled. The development environment uses GitLab.com, including updates to\nAnthropic Claude 3.5 as Large Language Model (LLM) for GitLab Duo [Code\nSuggestions](https://docs.gitlab.com/ee/user/gitlab_duo/#code-suggestions)\nand [Chat](https://docs.gitlab.com/ee/user/gitlab_duo/#gitlab-duo-chat).\nSpoiler: They are even more powerful and efficient.\n\n\nYou can navigate into each section of the article, or read top-down. The\nsource code and challenges with exercises are provided for self-learning,\ntoo.\n\n\n- [Refactor code to modern programming language\nstandards](#refactor-code-to-modern-programming-language-standards)\n    - [Generate Java 7 and refactor to Java 8](#generate-java-7-and-refactor-to-java-8)\n    - [Refactor across C++ standards](#refactor-across-c%2B%2B-standards)\n        - [Migration: Refactor C++03 into C++14](#migration-refactor-c%2B%2B03-into-c%2B%2B14)\n        - [Downgrade: Refactor C++23 to C++11](#downgrade-refactor-c%2B%2B23-to-c%2B%2B11)\n    - [Explain and refactor COBOL](#explain-and-refactor-cobol)\n- [Refactor a language into another\nlanguage](#refactor-a-language-into-another-language)\n    - [Refactor C to Rust](#refactor-c-to-rust)\n    - [Refactor Perl to Python](#refactor-perl-to-python)\n- [More refactoring hands-on](#more-refactoring-hands-on)\n    - [Refactor JavaScript](#refactor-javascript)\n    - [Refactor Bash to ZSH or SH](#refactor-bash-to-zsh-or-sh)\n    - [More use cases and tutorials](#more-use-cases-and-tutorials)\n- [Key takeaways](#key-takeaways)\n\n\n## Refactor code to modern programming language standards\n\n\nThe next sections show how to refactor, explain, and understand the language\nspecific differences in a defined programming language. \n\n\n### Generate Java 7 and refactor to Java 8 \n\n\nA customer wanted to know if GitLab Duo supports Java 7. While testing the\nCode Suggestions and Chat prompts to generate Java 7 specific code, I also\nthought of migration help. Anthropic Claude 3.5 is capable of understanding\nthe language-specific differences, and so is GitLab Duo.\n\n\nThe following example shows a prompt to generate Java 7 code with Code\nSuggestions.\n\n\n```java\n\n// Generate a new class for a File Manager. Use Java 7 features only.\n\nimport java.io.*;\n\nimport java.nio.file.*;\n\nimport java.util.*;\n\n\npublic class FileManager {\n    private Path rootDirectory;\n\n    public FileManager(String rootPath) throws IOException {\n        this.rootDirectory = Paths.get(rootPath);\n        if (!Files.exists(rootDirectory)) {\n            Files.createDirectories(rootDirectory);\n        }\n    }\n\n    public void createFile(String fileName, String content) throws IOException {\n        Path filePath = rootDirectory.resolve(fileName);\n        Files.write(filePath, content.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n    }\n\n    public String readFile(String fileName) throws IOException {\n        Path filePath = rootDirectory.resolve(fileName);\n        return new String(Files.readAllBytes(filePath));\n    }\n\n    public void deleteFile(String fileName) throws IOException {\n        Path filePath = rootDirectory.resolve(fileName);\n        Files.delete(filePath);\n    }\n\n    public List\u003CString> listFiles() throws IOException {\n        List\u003CString> fileList = new ArrayList\u003C>();\n        try (DirectoryStream\u003CPath> stream = Files.newDirectoryStream(rootDirectory)) {\n            for (Path file : stream) {\n                fileList.add(file.getFileName().toString());\n            }\n        }\n        return fileList;\n    }\n\n    public void copyFile(String sourceFileName, String destinationFileName) throws IOException {\n        Path sourcePath = rootDirectory.resolve(sourceFileName);\n        Path destinationPath = rootDirectory.resolve(destinationFileName);\n        Files.copy(sourcePath, destinationPath, StandardCopyOption.REPLACE_EXISTING);\n    }\n\n    public void moveFile(String sourceFileName, String destinationFileName) throws IOException {\n        Path sourcePath = rootDirectory.resolve(sourceFileName);\n        Path destinationPath = rootDirectory.resolve(destinationFileName);\n        Files.move(sourcePath, destinationPath, StandardCopyOption.REPLACE_EXISTING);\n    }\n\n    public long getFileSize(String fileName) throws IOException {\n        Path filePath = rootDirectory.resolve(fileName);\n        return Files.size(filePath);\n    }\n\n    public boolean fileExists(String fileName) {\n        Path filePath = rootDirectory.resolve(fileName);\n        return Files.exists(filePath);\n    }\n}\n\n```\n\n\nYou can continue with the following steps in your IDE:\n\n\n1. Learn and understand the generated code: Select the code in the editor\n(blue color in the screenshot), switch into GitLab Duo Chat and send the\n`/explain` slash command as prompt.\n\n\n![Validate the generated code with the `/explain` slash command in GitLab\nDuo\nChat](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/intellij_java7_generate_refactor.png)\n\n\n2. Refactor the Java 7 code into Java 8: Select the code in the editor,\nswitch into Chat, and send `/refactor using Java 8 features` as refined\nprompt.\n\n3. Practice an alternative to refactoring: Create a new file `java8.java`\nand generate Java 8 specific code using Code Suggestions with the prompt `//\nGenerate a new class for a File Manager. Use Java 8 features only.`.\n\n4. Continue writing Java 8 code in the same context with AI-powered code\ncompletions.\n\n\nYou can watch all steps in this recording.\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/XKRv6uBkD2I\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nThe source code is available in the [GitLab Duo Challenge - Generate and\nrefactor Java 7 to 8\nproject](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-generate-refactor-java-7-to-8).\n\n\n### Refactor across C++ standards\n\n\nThe C++ standard has a long history and can be considered stable and mature.\nDepending on the target architecture and platforms, the supported standard\ndepends on the compiler on the host operating system.\n\n\nIn my career, I needed to support 25 different Linux and Windows\ndistributions in open source projects, and have seen varying compiler\nsupport for newer language features and C++ standards. In 2015, we settled\non C++11 features, and later never touched based on C++14 or C++17. \n\n\nRefactoring functions and algorithms to a new C++ standard would have\nrequired too much effort, and help explaining the differences was needed.\nBack then, we researched through books, online search, Stack Overflow, and\nsometimes tools assistance for refactoring.\n\n\nThings have changed with AI-powered assistants and GitLab Duo: You can\ninstruct Code Suggestions and Chat to provide source code in a specific C++\nstandard. Then, it is also possible to compare the different outputs. \n\n\nThe following example uses a prompt to specifically generate source code for\nthe C++03 standard. It implements a simple config parser for the `.ini`\nconfiguration format.\n\n\n```c++\n\n/* Create a config parser which can parse an ini format\n\nUse C++03 standard only\n */\n\n#include \u003Ciostream>\n\n#include \u003Cfstream>\n\n#include \u003Cstring>\n\n#include \u003Cmap>\n\n#include \u003Csstream>\n\n\nclass ConfigParser {\n\nprivate:\n    std::map\u003Cstd::string, std::map\u003Cstd::string, std::string> > config;\n\npublic:\n    ConfigParser() {}\n\n    bool parse(const std::string& filename) {\n        std::ifstream file(filename.c_str());\n        if (!file.is_open()) {\n            return false;\n        }\n\n        std::string line;\n        std::string currentSection;\n\n        while (std::getline(file, line)) {\n            // Trim whitespace\n            size_t first = line.find_first_not_of(\" \\t\");\n            size_t last = line.find_last_not_of(\" \\t\");\n            if (first == std::string::npos) continue; // Skip empty lines\n            line = line.substr(first, (last - first + 1));\n\n            if (line[0] == '[' && line[line.length() - 1] == ']') {\n                // This is a section\n                currentSection = line.substr(1, line.length() - 2);\n            } else {\n                // This is a key-value pair\n                size_t delimiterPos = line.find('=');\n                if (delimiterPos != std::string::npos) {\n                    std::string key = line.substr(0, delimiterPos);\n                    std::string value = line.substr(delimiterPos + 1);\n                    config[currentSection][key] = value;\n                }\n            }\n        }\n\n        file.close();\n        return true;\n    }\n\n    std::string getValue(const std::string& section, const std::string& key) const {\n        std::map\u003Cstd::string, std::map\u003Cstd::string, std::string> >::const_iterator sectionIt = config.find(section);\n        if (sectionIt != config.end()) {\n            std::map\u003Cstd::string, std::string>::const_iterator keyIt = sectionIt->second.find(key);\n            if (keyIt != sectionIt->second.end()) {\n                return keyIt->second;\n            }\n        }\n        return \"\";\n    }\n\n    void printConfig() const {\n        std::map\u003Cstd::string, std::map\u003Cstd::string, std::string> >::const_iterator sectionIt;\n        for (sectionIt = config.begin(); sectionIt != config.end(); ++sectionIt) {\n            std::cout \u003C\u003C \"[\" \u003C\u003C sectionIt->first \u003C\u003C \"]\" \u003C\u003C std::endl;\n            std::map\u003Cstd::string, std::string>::const_iterator keyIt;\n            for (keyIt = sectionIt->second.begin(); keyIt != sectionIt->second.end(); ++keyIt) {\n                std::cout \u003C\u003C keyIt->first \u003C\u003C \" = \" \u003C\u003C keyIt->second \u003C\u003C std::endl;\n            }\n            std::cout \u003C\u003C std::endl;\n        }\n    }\n};\n\n\nint main() {\n    ConfigParser parser;\n    if (parser.parse(\"config.ini\")) {\n        std::cout \u003C\u003C \"Config file parsed successfully.\" \u003C\u003C std::endl;\n        parser.printConfig();\n\n        std::cout \u003C\u003C \"Value of 'key1' in 'section1': \" \u003C\u003C parser.getValue(\"section1\", \"key1\") \u003C\u003C std::endl;\n    } else {\n        std::cout \u003C\u003C \"Failed to parse config file.\" \u003C\u003C std::endl;\n    }\n    return 0;\n}\n\n```\n\n\nThe same prompts can be applied to [explaining\ncode](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#explain-code-in-the-ide),\n[generating\ntests](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#write-tests-in-the-ide),\nand even [refactoring\ncode](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#refactor-code-in-the-ide):\n`/explain focus on C++03 features`, `/tests for C++03 features`, `/refactor\ninto C++03`. \n\n\nThe following C++ standards have been tested working with GitLab Duo Code\nSuggestions:\n\n\n- C++03\n\n- C++11\n\n- C++14\n\n- C++17\n\n- C++20\n\n- C++23\n\n\nYou can find the source code in the [Challenge - Refactor Cpp Standards\nConfig Parser\nproject](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-cpp-standards-config-parser). \n\n\nThere are many different combinations to refactor source code between the\nC++ standards. The following examples reflect on modernizing the C++ code\nwith version upgrades, but also test a downgrade where ambitious developers\nwanted to use C++20 but had to use C++11 to support enterprise customer\nenvironments.\n\n\n#### Migration: Refactor C++03 into C++14 \n\n\nThe goal is to modernize the code, and understand the changes between C++03\nand C++14. Open the\n[03.cpp](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-cpp-standards-config-parser/-/blob/main/03.cpp?ref_type=heads)\nsource code in the IDE, select it, and use the following Chat prompt: \n\n\n```markdown\n\n/refactor into C++14 and explain the changes\n\n```\n\n\nPotential Chat response: \n\n\n```markdown\n\n…\n\n\nExplanations of the changes:\n\n\n1. Default constructor: Changed `ConfigParser() {}` to `ConfigParser() =\ndefault;` to use the default constructor.\n\n2. File opening: Removed `.c_str()` when opening the file, as C++11 and\nlater support `std::string` directly.\n\n3. Lambda functions: Introduced a lambda function trim to handle string\ntrimming, making the code more readable and reusable.\n\n4. Auto keyword: Used `auto` for type deduction in various places, improving\ncode readability.\n\n5. Range-based for loops: In the `printConfig` method, replaced traditional\nfor loops with range-based for loops, which are more concise and less\nerror-prone.\n\n6. Structured bindings: Used structured bindings in the range-based for\nloops to directly access key-value pairs.\n\n7. If statements with initializers: In the `getValue` method, used if\nstatements with initializers to combine the find operation and the check in\na single line.\n\n8. String member functions: Used `front()` and `back()` instead of\narray-style access for the first and last characters of strings.\n\n9. Uniform initialization: Used uniform initialization `{}` instead of\nparentheses `()` for object creation.\n\n10. Removed `std::endl`: Replaced `std::endl` with `'\\n'` for better\nperformance, as `std::endl` forces a flush of the output buffer.\n\n\nThese changes make the code more modern, concise, and potentially more\nefficient while maintaining the same functionality.\n\n\n```\n\n\n![Changes reflected in\ncode](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/duo_refactor_cpp_clion_chat_cpp03_to_14_01.png)\n\n\n![Changes reflected in code\n2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675060/Blog/Content%20Images/duo_refactor_cpp_clion_chat_cpp03_to_14_02.png)\n\n\n#### Downgrade: Refactor C++23 to C++11\n\n\nSometimes, developers started a new feature proof-of-concept in a modern C++\nstandard like C++20. The actual supported versions for enterprise customer\nenvironments require older compilers with C++11 support. Downgrading the\nversions come at a cost, and require evaluating which language features are\nstill supported, and which other disadvantages could come across.\n\n\nOpen the\n[23.cpp](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-cpp-standards-config-parser/-/blob/main/23.cpp?ref_type=heads)\nsource code in the IDE, select it, and use the following Chat prompt: \n\n\n```markdown\n\n/refactor into C++11, explain the necessary changes and potential problems\n\n```\n\n\nPotential Duo Chat response: \n\n\n```markdown\n\nThe main changes required to make this code compatible with C++11 are:\n\n\n1. Remove the C++23 standard comment, as it's no longer applicable.\n\n2. Replace `std::string::find_first_not_of` and\n`std::string::find_last_not_of` with C++11 compatible alternatives in the\ntrim function.\n\n3. Use `const auto&` instead of `auto` for loop variables to avoid\nunnecessary copying.\n\n\nPotential problems in the original code:\n\n\n1. The original code used C++23 features, which might not be widely\nsupported yet.\n\n2. The `trim` function used `std::string::find_first_not_of` and\n`std::string::find_last_not_of`, which are not available in C++11.\n\n3. The use of `auto` instead of `const auto&` in some loops could lead to\nunnecessary copying of complex objects.\n\n\nThese issues have been addressed in the refactored version. The `trim`\nfunction now uses `std::find_if_not` with lambda functions to achieve the\nsame result in a C++11 compatible way. The use of `const auto&` ensures that\nno unnecessary copying occurs when iterating over the config map.\n\n```\n\n\n![The `trim` function now uses `std::find_if_not` with lambda functions to\nachieve the same result in a C++11 compatible\nway](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/duo_refactor_cpp_clion_chat_cpp23_to_11_01.png)\n\n\n![The use of `const auto&` ensures that no unnecessary copying occurs when\niterating over the config\nmap](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/duo_refactor_cpp_clion_chat_cpp23_to_11_02.png)\n\n\n**Async practice**: Test more version refactoring scenarios.\n\n\n### Explain and refactor COBOL\n\n\nYou can use GitLab Duo to explain the source code, analyze, fix and refactor\nfor COBOL programs. I have never written nor learned COBOL, and found this\nhelpful [COBOL Programming\nCourse](https://github.com/openmainframeproject/cobol-programming-course)\nwith many examples.\n\n\nI then asked Chat how to get started with COBOL, create a COBOL program, and\ncompile a COBOL program on macOS.\n\n\n```markdown\n\nPlease explain what COBOL is and its syntax\n\n\nPlease create a COBOL program that shows the first steps\n\n\nTell me more about the COBOL compiler. Which system do I need? Can I do it\non my macOS?\n\n\n```\n\n\n![Asking GitLab Duo Chat to explain and its\nsyntax](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/vscode_chat_cobol_generate_example.png)\n\n\nOpen a COBOL program, select the source code, switch to Duo Chat and send\nthe `/explain` prompt to explain purpose and functionality.\n\n\nYou can also refine the prompts to get more high-level summaries, for\nexample:\n\n\n```markdown \n\n/explain like I am five\n\n```\n\n\n> Tip: Programming languages share similar algorithms and functionality. For\nCOBOL, Chat offered to explain it using Python, and, therefore, I adjusted\nfuture prompts to ask for an explanation in Python.\n\n\n```markdown\n\n/explain in a different programming language\n\n```\n\n\nYou can also use the `/refactor` slash command prompt in Chat to improve the\ncode quality, fix potential problems, and try to refactor COBOL into Python.\n\n\n```markdown\n\n/refactor fix the environment error\n\n\n/refactor fix potential problems\n\n\n/refactor into Python\n\n```\n\n\nThe [GitLab Duo Coffee Chat - Challenge: Explain and Refactor COBOL\nprograms](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-explain-refactor-cobol-program)\nrecording shows all discussed steps in a practical use case, including how\nto find a missing period: \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/pwlDmLQMMPo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Refactor a language into another language\n\n\nModernization and code quality improvements sometimes require the change of\na programming language. Similar refactor prompts with GitLab Duo can help\nspeed up the migration process. The COBOL example with Python is just one of\nmany requirements in enterprise environments -- let's dive into more use\ncases.\n\n\n### Refactor C to Rust \n\n\nIn early 2024, several programming languages, like C, have been called out\nfor not being memory safe. The recommendations for future projects include\n[memory safe\nlanguages](https://about.gitlab.com/blog/memory-safe-vs-unsafe/)\nlike Rust. But how do you start a migration, and what are the challenges?\n\n\nLet's try it with a simple example in C. The code was generated using Code\nSuggestions and should print the basic operating system information, like\nthe name, version, and platform. The C code compiles cross-platform on\nWindows, Linux, and macOS.\n\n\n```c\n\n// Read OS files to identify the platform, name, versions\n\n// Print them on the terminal\n\n#include \u003Cstdio.h>\n\n#include \u003Cstdlib.h>\n\n#include \u003Cstring.h>\n\n\n#ifdef _WIN32\n    #include \u003Cwindows.h>\n#elif __APPLE__\n    #include \u003Csys/utsname.h>\n#else\n    #include \u003Csys/utsname.h>\n#endif\n\n\nvoid get_os_info() {\n    #ifdef _WIN32\n        OSVERSIONINFOEX info;\n        ZeroMemory(&info, sizeof(OSVERSIONINFOEX));\n        info.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);\n        GetVersionEx((OSVERSIONINFO*)&info);\n\n        printf(\"Platform: Windows\\n\");\n        printf(\"Version: %d.%d\\n\", info.dwMajorVersion, info.dwMinorVersion);\n        printf(\"Build: %d\\n\", info.dwBuildNumber);\n    #elif __APPLE__\n        struct utsname sys_info;\n        uname(&sys_info);\n\n        printf(\"Platform: macOS\\n\");\n        printf(\"Name: %s\\n\", sys_info.sysname);\n        printf(\"Version: %s\\n\", sys_info.release);\n    #else\n        struct utsname sys_info;\n        uname(&sys_info);\n\n        printf(\"Platform: %s\\n\", sys_info.sysname);\n        printf(\"Name: %s\\n\", sys_info.nodename);\n        printf(\"Version: %s\\n\", sys_info.release);\n    #endif\n}\n\n\nint main() {\n    get_os_info();\n    return 0;\n}\n\n```\n\n\nOpen the source code in\n[`os.c`](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-c-to-rust/-/blob/897bf57a14bb7be07d842e7f044f93a61456d611/c/os.c)\nin JetBrains CLion, for example. Select the source code and use the Chat\nprompt `/explain` to explain purpose and functionality. Next, use\n`/refactor` in the Chat prompt to refactor the C code, and then take it one\nstep further: `/refactor into Rust`. \n\n\nInitialize a new Rust project (Tip: Ask Duo Chat), and copy the generated\nsource code into the `src/main.rs` file. Run `cargo build` to compile the\ncode. \n\n\n![Initialize a new Rust project, and copy the generated source code into the\n`src/main.rs` file. Run `cargo build` to compile the\ncode.](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/jetbrains_clion_c_rust.png)\n\n\nIn the [GitLab Duo Coffee Chat: Challenge - Refactor C into\nRust](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-c-to-rust)\nrecording, you can learn all steps, and additionally, you'll see a\ncompilation error which gets fixed with the help of Chat and `/refactor`\nslash command. The session also shows how to improve the maintanability of\nthe new Rust code by adding more error handling. \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/nf8g2ucqvkI\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n### Refactor Perl to Python \n\n\nThat one script that runs on production servers, does its job, the author\nleft the company ten years ago, and nobody wants to touch it. The problem\nmight also apply to multiple scripts, or even a whole application. A\ndecision was made to migrate everything to modern Python 3, with the goal to\nmodernize the code, and understand the changes between Perl and Python.\n\n\nA customer recently asked in a GitLab Duo workshop whether a direct\nmigration is possible using GitLab Duo. Short answer: Yes, it is. Longer\nanswer: You can use refined Chat prompts to refactor Perl code into Python,\nsimilar to other examples in this article.\n\n\nOpen the `script.pl` source code in IDE, select it, and open Chat.\n\n\n```perl\n\n#!/usr/bin/perl\n\nuse strict;\n\nuse warnings;\n\n\nopen my $md_fh, '\u003C', 'file.md' or die \"Could not open file.md: $!\";\n\n\nmy $l = 0;\n\nmy $e = 0;\n\nmy $h = 0;\n\n\nwhile (my $line = \u003C$md_fh>) {\n  $l++;\n  if ($line =~ /^\\s*$/) {\n    $e++;\n    next;\n  }\n  if ($line =~ /^#+\\s*(.+)/) {\n    print \"$1\\n\";\n    $h++; \n  }\n}\n\n\nprint \"\\nS:\\n\"; \n\nprint \"L: $l\\n\";\n\nprint \"E: $e\\n\"; \n\nprint \"H: $h\\n\";\n\n```\n\n\nYou can use the following prompts to:\n\n\n1. `/explain` its purpose, and `/refactor` to improve the code.\n\n2. `/refactor into Python` to get a working Python script.\n\n\n![Refactor into\nPython](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/pycharm_duo_refactor_perl_python.png)\n\n\n> Tip: You can refactor Perl code into more target languages. The [GitLab\nDuo Coffee Chat: Challenge - Refactor Perl to\nPython](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-perl-python)\nrecording shows PHP, Ruby, Rust, Go, Java, VB.NET, C#, and more.\n\n> \n\n> If you want to continue using Perl scripts, you can configure [Perl as\nadditional\nlanguage](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html#add-support-for-more-languages)\nin Duo Code Suggestions. Chat already understands Perl and can help with\nquestions and slash command prompts, as you can see in the following\nrecording.\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/03HGhxXg9lw\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## More Refactoring Hands-on \n\n\n### Refactor JavaScript \n\n\nEddie Jaoude shows how to refactor JavaScript to improve code quality or add\nfunctionality in a practical example. \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/mHn8KOzpPNY\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n### Refactor Bash to ZSH or SH\n\n\nI have used Bash as a shell for 20 years and most recently switched to ZSH\non macOS. This resulted in script not working, or unknown errors in my\nterminal. Another use case for refactoring are shell limitations – some\noperating systems or Linux/Unix distributions do not provide Bash, only SH,\nfor example, Alpine.\n\n\n![Refactor shell\nscripts](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675059/Blog/Content%20Images/intellj_refactor_shell_scripts.png)\n\n\nThe [GitLab Duo Coffee Chat: Challenge - Refactor Shell\nScripts](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/code-challenges/challenge-refactor-shell-scripts)\nshows an example with a C program that can tail syslog files, and a build\nscript written in Bash. Throughout the challenge, Chat is queried with\n`/explain` and `/refactor` prompts to improve the code. It is also possible\nto refactor Bash into POSIX-compliant SH or ZSH. The session concludes with\nasking Chat to provide five different Shell script implementations, and\nexplain the key summaries. \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/mssqYjlKGzU\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n### More use cases and tutorials\n\n\n- [Documentation: GitLab Duo use\ncases](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html)\n\n- [Tutorial: Top tips for efficient AI-powered code suggestions with GitLab\nDuo](https://about.gitlab.com/blog/top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo/)\n\n- [Tutorial: 10 best practices for using AI-powered GitLab Duo\nChat](https://about.gitlab.com/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat/)\n\n\n## Key takeaways \n\n\n1. GitLab Duo provides efficient help with explaining and refactoring code. \n\n1. You can refactor code between language standards, and ask follow-up\nquestions in Chat.\n\n1. Code Suggestions prompts can generate specific language standards, and\ncode completion respects the current code context. \n\n1. Refactoring code into new programming languages helps with longer term\nmigration and modernization plans.\n\n1. Code can be \"downgraded\" into older system's supported language\nstandards.\n\n1. GitLab Duo can explain complex code and programming languages with\ndifferent programming language examples.\n\n1. The update to Anthropic Claude 3.5 on GitLab.com has improved the quality\nand speed of Code Suggestions and Chat once again (self-managed upgrade to\n17.3 recommended).\n\n1. There are no boundaries except your imagination, and production pain\npoints.\n\n\nLearn more about efficient Code Suggestions and Chat workflows, and start\nyour AI-powered code refactoring journey with GitLab Duo today!\n\n\n> [Start your free trial of GitLab\nDuo!](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/?type=free-trial&toggle=gitlab-duo-pro_)\n",[720,9,851],{"slug":3688,"featured":6,"template":700},"refactor-code-into-modern-languages-with-ai-powered-gitlab-duo","content:en-us:blog:refactor-code-into-modern-languages-with-ai-powered-gitlab-duo.yml","Refactor Code Into Modern Languages With Ai Powered Gitlab Duo","en-us/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo.yml","en-us/blog/refactor-code-into-modern-languages-with-ai-powered-gitlab-duo",{"_path":3694,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3695,"content":3701,"config":3706,"_id":3708,"_type":13,"title":3709,"_source":15,"_file":3710,"_stem":3711,"_extension":18},"/en-us/blog/refactoring-a-ci-cd-template-to-a-ci-cd-component",{"title":3696,"description":3697,"ogTitle":3696,"ogDescription":3697,"noIndex":6,"ogImage":3698,"ogUrl":3699,"ogSiteName":685,"ogType":686,"canonicalUrls":3699,"schema":3700},"Refactoring a CI/CD template to a CI/CD component","CI/CD components are the next generation of CI/CD templates, enhancing pipeline creation and maintenance. Learn how to transition from templates to components.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749665989/Blog/Hero%20Images/AdobeStock_618473457.jpg","https://about.gitlab.com/blog/refactoring-a-ci-cd-template-to-a-ci-cd-component","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Refactoring a CI/CD template to a CI/CD component\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2024-03-04\",\n      }",{"title":3696,"description":3697,"authors":3702,"heroImage":3698,"date":3703,"body":3704,"category":785,"tags":3705},[1096],"2024-03-04","GitLab recently introduced [CI/CD components](https://docs.gitlab.com/ee/ci/components/) as the next generation of the traditional CI/CD templates, and a novel approach to constructing CI/CD pipelines. CI/CD components offer reusable pipeline configurations that can be customized using input parameters.\n\nAlthough GitLab continues to support templates, they come with certain drawbacks that are addressed by the introduction of components. Therefore, we highly recommend refactoring existing templates into CI/CD components.\n\nThis article will guide you through the steps of converting your current GitLab CI/CD templates into reusable CI/CD components. Prior familiarity with how to create CI/CD components is a prerequisite, which you can learn about in this blog post: [Introducing the GitLab CI/CD Catalog Beta](https://about.gitlab.com/blog/introducing-the-gitlab-ci-cd-catalog-beta/).\n\n## How to convert a template to a component\n\nThese are the steps to convert a CI/CD template to a CI/CD component:\n1. Create a component project if you don’t have one. \n2. Copy your existing templates to the ‘templates’ directory in the component project. \n3. For each template, review the jobs listed in it and assess whether you prefer to distribute them across different components or retain some or all within the same component. While it's possible to include multiple jobs in a single component, it's advisable to create components that perform minimal tasks. This approach enhances ease of reuse and flexibility.\n4. Create a new section at the top of the configuration for the input parameters and meta data using the `spec` keyword. \n5. Replace any custom CI/CD variables and any other hard-coded values with [inputs](https://about.gitlab.com/blog/use-inputs-in-includable-files/) to maximize flexibility for consumption. Consider parameterizing elements such as stage, image, job name/job prefix, etc. \n6. Follow the [best practices](https://docs.gitlab.com/ee/ci/components/index.html#best-practices) for components.\n7. Improve the configuration, for example by enabling [merge request pipelines](https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html) or making it [more efficient](https://docs.gitlab.com/ee/ci/pipelines/pipeline_efficiency.html).\n\nHere is a code example of a job in an existing template:\n\n![existing template](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749678076/Blog/Content%20Images/Screenshot_2024-03-03_at_12.05.25.png)\n\nAnd this is the refactored [component code](https://gitlab.com/components/aws/-/blob/main/templates/ec2-deploy-production.yml?ref_type=heads):\n\n![Converted component](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749678077/Blog/Content%20Images/Screenshot_2024-03-03_at_12.07.14.png)\n\nOnce your components are ready, you can publish them to the CI/CD catalog so others will be able to find and consume them. \n\n## Take a product tour\n\nWe've prepared a brief product tour so you can quickly dive into the CI/CD catalog and see it in action (use the \"Next\" button to progress through the demo).\n\n[![Product tour of CI/CD catalog](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749678076/Blog/Content%20Images/Screenshot_2024-03-03_at_12.18.09.png)](https://gitlab.navattic.com/cicd-catalog)\n\n## Additional resources \n\nPlease refer to the official documentation on CI/CD components for more on how to [convert templates to components](https://docs.gitlab.com/ee/ci/components/#convert-a-cicd-template-to-a-component).\n\nYou can explore [an additional practical example](https://docs.gitlab.com/ee/ci/components/examples.html#cicd-component-migration-examples), demonstrating the steps to convert GitLab Go templates to CI/CD components.\n\nThen, you can watch the following video where [Fabio Pitino](https://about.gitlab.com/company/team/#fabiopitino), GitLab Principal Engineer, demonstrates the process of refactoring GitLab AWS templates to CI/CD components.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/dGCPrIAuBmE?si=1vjG_aEziY5jn-YC\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line –->\n",[108,851,9],{"slug":3707,"featured":90,"template":700},"refactoring-a-ci-cd-template-to-a-ci-cd-component","content:en-us:blog:refactoring-a-ci-cd-template-to-a-ci-cd-component.yml","Refactoring A Ci Cd Template To A Ci Cd Component","en-us/blog/refactoring-a-ci-cd-template-to-a-ci-cd-component.yml","en-us/blog/refactoring-a-ci-cd-template-to-a-ci-cd-component",{"_path":3713,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3714,"content":3720,"config":3726,"_id":3728,"_type":13,"title":3729,"_source":15,"_file":3730,"_stem":3731,"_extension":18},"/en-us/blog/refactoring-javascript-to-typescript-with-gitlab-duo-workflow",{"title":3715,"description":3716,"ogTitle":3715,"ogDescription":3716,"noIndex":6,"ogImage":3717,"ogUrl":3718,"ogSiteName":685,"ogType":686,"canonicalUrls":3718,"schema":3719},"Refactoring JavaScript to TypeScript with GitLab Duo Workflow","Learn how we used our autonomous AI agent, which sits in your development environment, to convert a real-world JavaScript application to TypeScript.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749660174/Blog/Hero%20Images/Workflow_1800x945.png","https://about.gitlab.com/blog/refactoring-javascript-to-typescript-with-gitlab-duo-workflow","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Refactoring JavaScript to TypeScript with GitLab Duo Workflow\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Frédéric Caplette\"}],\n        \"datePublished\": \"2025-05-22\",\n      }",{"title":3715,"description":3716,"authors":3721,"heroImage":3717,"date":3723,"body":3724,"category":718,"tags":3725},[3722],"Frédéric Caplette","2025-05-22","TypeScript adoption continues to grow, with over 88% of developers reporting\nthey either use or want to use it. Yet, migrating existing JavaScript\ncodebases to TypeScript is often a time-consuming process. Enter [GitLab Duo\nWorkflow](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/):\nsecure, agentic AI that sits right inside your development environment,\nhelping transform high-level tasks into executable workflows. In this\narticle, you'll learn how we used Duo Workflow to update Duo Workflow,\nconverting a real-world JavaScript application to TypeScript. We'll also\nreview the technical process and broader implications for development\nworkflows.\n\n\nThis video walks through visually what you'll read below:\n\n\n\u003Cdiv style=\"padding:56.25% 0 0 0;position:relative;\">\u003Ciframe\nsrc=\"https://player.vimeo.com/video/1085078036?badge=0&autopause=0&player_id=0&app_id=58479\"\nframeborder=\"0\" allow=\"autoplay; fullscreen; picture-in-picture;\nclipboard-write; encrypted-media\"\nstyle=\"position:absolute;top:0;left:0;width:100%;height:100%;\"\ntitle=\"Refactor JavaScript to TypeScript with GitLab Duo\nWorkflow\">\u003C/iframe>\u003C/div>\u003Cscript\nsrc=\"https://player.vimeo.com/api/player.js\">\u003C/script>\n\n\n## The challenge: Refactor JS to TS\n\n\nWe decided to migrate Duo Workflow client-related logic to TypeScript for\nbetter type safety and auto-complete. A JavaScript-to-TypeScript migration\ninvolves more than just changing file extensions. It requires:\n\n\n1. Analyzing existing code patterns to determine appropriate types\n\n2. Handling edge cases where type inference is ambiguous\n\n3. Ensuring consistency across the codebase\n\n4. Managing dependencies and third-party libraries\n\n5. Validating that runtime behavior remains unchanged\n\n\nDoing all of this work manually can be very time consuming and not much fun.\nThankfully, managing your projects is easier and more efficient with Duo\nWorkflow – even when the project is Duo Workflow.\n\n\n## How it works\n\n\nIf you've used AI coding assistants before, you're likely familiar with\ntheir chat-based interfaces (like [GitLab Duo\nChat](https://about.gitlab.com/blog/gitlab-duo-chat-now-generally-available/)).\nThese chats can provide code snippets, suggested directory structures, and\nmore. GitLab Duo Workflow fundamentally differs from these in several key\nways:\n\n\n1. It's contextually aware of your entire codebase, not just open files or\nhighlighted text. You can also add context from relevant GitLab issues or\nmerge requests.\n\n2. It develops and executes a multi-stage plan, allowing for more complete\nsolutions.\n\n3. It keeps a human in the loop, ensuring you understand and approve of what\nis happening at each step of the way.\n\n4. It stages changes to your repository, giving you control over what to\naccept.\n\n\nDuo Workflow uses an agentic AI approach, powered by Claude 3.5 Sonnet, to\nunderstand code semantics while maintaining a deep integration with GitLab's\nDevOps platform.\n\n\nLet's walk through a real-world example of refactoring a JavaScript file\nusing GitLab Duo Workflow. In our case, we started with the prompt:\n\n\n```unset\n\nI need to refactor a JavaScript file\npackages/webview_duo_workflow/src/app/stores/workflow.js to TypeScript.\nPlease analyze the code, identify any issues, and convert it to TypeScript\nwith appropriate type definitions.\n\n```\n\n\n### Analysis\n\n\nAfter prompting, Duo Workflow first analyzed the codebase structure:\n\n\n* Identified that `workflow.js` is a Pinia store implementation\n\n* Noted it uses imports from various modules including `constants.ts` (which\nwas already in TypeScript)\n\n* Recognized GraphQL queries import exists but the file wasn't accessible\n\n\nBased on this analysis, Duo Workflow identified specific typing needs:\n\n\n**Type definitions needed for:**\n\n\n* Store State interface with precise types:\n  * `areWorkflowsLoading`: boolean\n  * `isLoadingWorkflow`: boolean\n  * `isLoadingRecentWorkflows`: boolean\n  * `recentWorkflows`: array\n  * `workflows`: array\n  * `workflowsPageInfo`: object\n  * `initialState`: object\n  * `activeWorkflow`: object with specific shape\n\n**Interface definitions required:**\n\n\n* Workflow interface with properties:\n  * `id`: string\n  * `goal`: string\n  * `status`: string\n  * `checkpoint`: object\n  * `type`: string\n\n**Parameter types needed for:**\n\n\n* GraphQL response types for workflows data\n\n* Event handler parameter types\n\n* Function parameter types for actions like `getWorkflowById`,\n`runWorkflow`, etc.\n\n\nDuo Workflow identified several complex patterns requiring special handling:\n\n\n* Uses optional chaining extensively\n\n* Has nested object destructuring\n\n* Contains GraphQL integration\n\n* Uses Pinia store patterns with state, getters, and actions\n\n* Implements an events mapping system\n\n\nThe system recognized external type dependencies:\n\n\n* Pinia store types\n\n* GraphQL query response types\n\n* Functions from an internal Bridge module were also properly recognized\n(for `sendRequest`, `sendGraphqlRequest`)\n\n\nThe final analysis concluded: \"The file will need to be renamed from .js to\n.ts and appropriate type definitions added while maintaining the existing\nfunctionality.\"\n\n\n### Planning\n\n\nBased on this thorough analysis, Duo Workflow created a structured plan\nconsisting of specific tasks:\n\n\n1. **Create a new TypeScript file** `workflow.ts` in the same directory as\nthe original JS file.\n\n\n2. **Create TypeScript interfaces** for the store state and workflow types\nat the top of the new file, including:\n\n   * WorkflowState interface for store state\n   * Workflow interface for workflow object structure\n   * WorkflowPageInfo interface for pagination info\n   * WorkflowCheckpoint interface for checkpoint data\n\n3. **Copy and add type annotations** to the store implementation from\n`workflow.js`, including:\n\n   * State return type\n   * Getter return types\n   * Action parameter types\n   * Action return types\n\n4. **Update all imports** in `workflow.ts` to:\n\n   * Add type imports from Pinia\n   * Update local imports to use .ts extensions where applicable\n   * Import any required type definitions\n\n### Execution\n\n\nAfter the plan is complete, we were prompted to  “Approve plan.” Before\nclicking approve, we reviewed each step and ensured we were comfortable with\nthe plan. After approval, Duo Workflow showed its progress through each step\nwith visual indicators and explanations of what API operations were\nsupporting each task (like \"Supported by: create_file_with_contents\" or\n\"Supported by: edit\\_file\"). When the work was done, we reviewed the changes\nbefore committing.\n\n\n## What we learned\n\n\nThis JavaScript-to-TypeScript migration example showcases a significant\nevolution in AI-assisted development. What makes GitLab Duo Workflow\nparticularly interesting is its approach to:\n\n\n### Task-oriented programming vs. suggestion-only assistance\n\n\nUnlike many AI assistants that simply offer code snippets or suggestions,\nDuo Workflow understands and executes complete tasks. The difference is\nsignificant — rather than saying \"here's some TypeScript code you might\nuse,\" it says \"I'll convert this file for you, here's my plan, and here are\nthe changes I'm making.\"\n\n\n### Contextual understanding of the entire codebase\n\n\nThe tool demonstrates awareness of project structure, related files (like\nconstants.ts and GraphQL queries), and the relationships between components.\nThis contextual understanding allows for more sophisticated conversions than\nlocalized transformations.\n\n\n### Step-by-step execution with visibility\n\n\nThe plan-based approach, with clear steps and progress indicators, provides\ntransparency into what would otherwise be a black-box process. This allows\ndevelopers to understand what the AI is doing and how it's approaching the\nproblem.\n\n\n> GitLab Duo Workflow is currently available in private beta for GitLab\nUltimate customers. [Sign up for the waitlist\ntoday!](https://about.gitlab.com/gitlab-duo/agent-platform/)\n\n\n## Learn more\n\n\n- [Agentic AI guides and\nresources](https://about.gitlab.com/blog/agentic-ai-guides-and-resources/)\n\n- [GitLab Duo\nWorkflow](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/)\n\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n",[720,9,495,696],{"slug":3727,"featured":90,"template":700},"refactoring-javascript-to-typescript-with-gitlab-duo-workflow","content:en-us:blog:refactoring-javascript-to-typescript-with-gitlab-duo-workflow.yml","Refactoring Javascript To Typescript With Gitlab Duo Workflow","en-us/blog/refactoring-javascript-to-typescript-with-gitlab-duo-workflow.yml","en-us/blog/refactoring-javascript-to-typescript-with-gitlab-duo-workflow",{"_path":3733,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3734,"content":3739,"config":3743,"_id":3745,"_type":13,"title":3746,"_source":15,"_file":3747,"_stem":3748,"_extension":18},"/en-us/blog/remediating-vulnerabilities-with-insights-and-ai",{"title":3735,"description":3736,"ogTitle":3735,"ogDescription":3736,"noIndex":6,"ogImage":1853,"ogUrl":3737,"ogSiteName":685,"ogType":686,"canonicalUrls":3737,"schema":3738},"Remediating vulnerabilities with GitLab's security insights and AI","Learn how to leverage vulnerability insights and the Explain this Vulnerability AI feature to not only resolve a vulnerability, but also understand it.","https://about.gitlab.com/blog/remediating-vulnerabilities-with-insights-and-ai","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Remediating vulnerabilities with GitLab's security insights and AI\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2023-08-31\",\n      }",{"title":3735,"description":3736,"authors":3740,"heroImage":1853,"date":1324,"body":3741,"category":718,"tags":3742},[1444],"We recently introduced [GitLab Duo](https://about.gitlab.com/gitlab-duo/), a\ncomplete suite of AI capabilities to power your DevSecOps workflows. GitLab\nDuo's AI features not only enable you to write secure code faster, but also\nenhance productivity by providing helpful explanations and insights into\nyour code. For instance, you can harness the power of AI to prevent security\nbreaches. In this tutorial, we will go over the Explain this Vulnerability\nAI feature, which is in beta, and how it can be used with vulnerability\ninsights to remediate vulnerabilities.\n\n\nYou will learn the following:\n\n* How the Explain this Vulnerability AI feature works\n\n* Prerequisites for Explain this Vulnerability and other GitLab AI features\n\n* How GitLab Vulnerability Insights assists in remediation\n\n* How to remediate a SQL-injection vulnerability using GitLab's\nvulnerability insights and Explain this Vulnerability\n\n* Additional GitLab AI capabilities (GitLab Duo currently requires\nconnectivity to access Google large language models (LLMs), however, there\nare plans to expand these features to limited-connectivity environments)\n\n\nSee the following video for a quick overview of Vulnerability Insights + AI\n\"Explain this Vulnerability\". \n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/1UagZx_CUks\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nYou can also see a detailed walkthrough of [Leveraging GitLab Vulnerability\nInsights + AI to Remediate a\nSQL-Injection](https://youtu.be/EJXAIzXNAWQ?feature=shared) in the [Solving\na SQL injection using vulnerability insights and\nAI](#solving-a-sql-injection-using-vulnerability-insights-and-ai) section\nbelow.\n\n\n## What is the Explain this Vulnerability AI feature?\n\nThe [Explain this\nVulnerability](https://docs.gitlab.com/ee/user/ai_features.html#explain-this-vulnerability-in-the-web-ui)\nfeature\n\nleverages an LLM powered by Google AI to assist you in securing your\napplication by:\n\n* Summarizing detected vulnerabilities\n\n* Helping developers and security analysts understand the vulnerability and\nits implications\n\n* Showing how a vulnerability can be exploited with detailed example code\n\n* Providing in-depth solutions to the vulnerability\n\n* Providing suggested mitigation along with sample code tuned toward your\nproject's programming language\n\n\nTo begin using Explain this Vulnerability, you must have the following\nprerequisites configured:\n\n\n* [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/) SaaS\nsubscription\n\n* [Experiment features\nenabled](https://docs.gitlab.com/ee/user/group/manage.html#enable-experiment-features)\n\n* [Third-party AI features\nenabled](https://docs.gitlab.com/ee/user/group/manage.html#enable-third-party-ai-features)\n\n* Static application security testing\n([SAST](https://docs.gitlab.com/ee/user/application_security/sast/))\nvulnerability finding in the default branch of a project\n\n* [Maintainer](https://docs.gitlab.com/ee/user/permissions.html) or greater\nrole in the vulnerable project \n\n* [SAST scanner](https://docs.gitlab.com/ee/user/application_security/sast/)\nenabled in the vulnerable project\n\n* An active internet connection\n\n\nOnce the prerequisites have been configured, to begin using Explain this\nVulnerability, perform the following steps:  \n\n\n1) Navigate to the [Vulnerability\nReport](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/).  \n\n2) Find a SAST vulnerability finding.  \n\n3) Scroll to the bottom of the [vulnerability\npage](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/).  \n\n4) Press the **Try it out** button in \"Explain this Vulnerability and how to\nmitigate it with AI\" section.  \n\n\n![View of the \"Try it out\" button at bottom of\nscreen](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/ai_explain_this_vulnerability_try_it_out_dialog.png)\n\n\nOnce you click the button, GitLab will begin to generate the following:\n\n* **What is the vulnerability?**: Details on the vulnerability and how it\nmay affect your application\n\n* **How can an attacker take advantage of the vulnerability?**: Commands\nthat a malicious actor can use to exploit the vulnerability\n\n* **How can the vulnerability be fixed?**: Details on how the vulnerability\ncan be remediated\n\n* **Example of vulnerable code**: The actual vulnerable code in the language\nof your application\n\n* **Example of fixed code**: Code showing a fix that should be applied to\nremediate the vulnerability in the language of your application\n\n* **References**: Links providing details relevant to the vulnerability\n\n* **User rating request**: Allows for user input, which is used to improve\nthe model\n\n\n![AI response depicting the above\nlist](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/ai_explain_this_vulnerability_results.png)\n\n\nThis information can be used together with vulnerability insights to resolve\nthe vulnerability. Now let's discuss vulnerability insights.\n\n\n## Vulnerability insights\n\nVulnerability insights provide detailed information on a vulnerability and\nhow to resolve it. This detailed information\n\nincludes:\n\n\n* **Description**: A detailed description of the vulnerability and its\nimplications\n\n* **Severity**: The severity of the vulnerability based on the [CVSS\nrating](https://nvd.nist.gov/vuln-metrics/cvss)\n\n* **Project**: The project where the vulnerability was found\n\n* **Tool**: The type of scanner that found the vulnerability\n\n* **Scanner**: The specific name of the scanner that found the vulnerability\n\n* **Location**: The line of code where the vulnerability is present\n\n* **Identifiers**: Links that identify and provide additional information on\nthe vulnerability such as the CVE/CWE page\n\n* **Training**: Security training available from our partners to educate\ndevelopers on the vulnerability\n\n* **Solution**: Information on how to remediate the vulnerability\n\n* **Method**: The [REST API\nmethod](https://www.w3schools.in/restful-web-services/rest-methods) used to\nexploit the vulnerability (dynamic scanners only)\n\n* **URL**: The URL in which the vulnerability was detected (dynamic scanners\nonly)\n\n* **Request/response**: The request sent and response received when\nexploiting the vulnerability (dynamic scanners only)\n\n\n**Note**: Results may vary depending on the scanner used.\n\n\nHaving all this information not only allows you to resolve a vulnerability\nwith ease but also enhances your security\n\nknowledge. All these insights are provided as a single source of truth that\nboth developer and security teams can view and\n\ntake action on asynchronously.\n\n\nDevelopers can leverage insights within a merge request (MR). The MR\ninsights show the vulnerabilities in the diff\n\nbetween a feature branch and the branch you are merging into. This allows\nyou to continuously iterate until you have resolved\n\na vulnerability and then alert security engineers when approval is required,\ngiving developers the power to resolve\n\nvulnerabilities themselves.\n\n\n![MR insights\nsample](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/vulnerability_insights_mr_view.png)\n\n\nThe security team can leverage insights via the [vulnerability\nreport](https://docs.gitlab.com/ee/user/application_security/vulnerability_report/).\nThe vulnerability report shows vulnerabilities present in the `default`\nbranch, which is typically linked to production. From here, the security\nteam can collaborate on a resolution as well as triage and manage\nvulnerabilities.\n\n\n![Vulnerability report\nsample](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/vulnerability_insights_vulnerability_report.png)\n\n\n**Note**: Currently, the Explain this Vulnerability feature can only be seen\nin the Vulnerability Report view. It is currently\n\nbeing considered for the MR view, see [future iterations under\nconsideration](https://gitlab.com/groups/gitlab-org/-/epics/10284#future-iterations-under-consideration)\nfor more information.\n\n\n## Solving a SQL injection using vulnerability insights and AI\n\nBy leveraging both vulnerability insights and Explain this Vulnerability, we\nhave all the resources necessary to\n\nnot only resolve a vulnerability but also understand it. Let's see how we\ncan use these features to [solve a SQL\ninjection](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/documentation/anatomy_of_a_vulnerability/). \n\n\nNow let's go over the steps to remediate a SQL injection. You can follow\nalong with the video:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/EJXAIzXNAWQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n**Privacy notice**: Explain this Vulnerability only uses `public repos` to\ntrain the LLM. Code in private repositories\n\nis not transferred to the LLM.\n\n\nI will be using the [Simple Notes\nproject](https://gitlab.com/gitlab-de/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\nto showcase this. You can set up DevSecOps within GitLab yourself by going\nover the following\n[tutorial](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/).\nAfter you have done so, you can run through the following:\n\n\n1) Navigate to **Secure > Vulnerability Report**.\n\n\n2) Sort by **SAST** under **Scanner**.\n\n\n3) Find and select a SQL injection vulnerability. a SQL injection will be\ntitled something like\n\n`Improper Neutralization of Special Elements used in an SQL Command ('SQL\nInjection')`.\n\n\n4) Examine the vulnerability insights.\n\n\n* **Description**: Detected possible formatted SQL query.\n\n* **Location**: File:\n[notes/db.py:100](https://gitlab.com/gitlab-de/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/-/blob/24ff1847aa70c4d51482fe28f019e3724b399aaf/notes/db.py#L100)\n\n* **Identifier**: [bandit.B608](https://semgrep.dev/r/gitlab.bandit.B608),\n[CWE-89](https://cwe.mitre.org/data/definitions/89.html)\n\n* **Solution**: Use parameterized queries instead=\n\n* **Training**: [Secure Code\nWarrior](https://portal.securecodewarrior.com/?utm_source=partner-integration:gitlab&partner_id=gitlab#/contextual-microlearning/web/injection/sql/python/vanilla),\n[SecureFlag](https://knowledge-base.secureflag.com/vulnerabilities/sql_injection/sql_injection_python.html),\nand\n[Kontra](https://application.security/gitlab/free-application-security-training/owasp-top-10-sql-injection)\n\n\n![SQL Injection Walkthrough -\nInsights](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/vulnerability_insights_vulnerability_report.png)\n\n\n5) Scroll down to the \"Explain this vulnerability and how to mitigate it\nwith AI** section and click the **Try it out** button.\n\n\n**Privacy notice**: If the **Send code to prompt** radio button is selected,\nresponse quality is improved. However, the actual code is\n\nused in a query to the LLM (even in private repositories).\n\n\n![SQL Injection Walkthrough - AI \"Try it out\"\nbutton](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/ai_explain_this_vulnerability_try_it_out_dialog.png)\n\n\n6) Examine the provided AI solutions.\n\n\n![SQL Injection Walkthrough - AI\nresponse](https://about.gitlab.com/images/blogimages/2023-08-31-solving-vulnerabilities-with-insights-and-ai/ai_explain_this_vulnerability_results.png)\n\n\n7) Exploit the vulnerability\n\nWe can use the information provided in the **AI response**, the samples in\nthe **vulnerability insight CWE identifier**,\n\nand the applications [API\nguide](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/documentation/api_guide/)\nto generate a malicious curl command as follows:\n\n\n```bash\n\n# A REGULAR API-CALL\n\n$ curl http://{LOAD_BALANCER_IP}/{APPLICATION_PATH}/api\n\n\n{\"Note\":\"[(1, 'cat'), (2, 'dog'), (3, 'frog'), (4, 'hog')]\"}\n\n\n# API CALL PASSING '1 or 1=1' AS SHOWN IN AI RESPONSE AND DETAILED IN\nIDENTIFIERS\n\n# NOTE: `1%20or%201%3D1` IS URL ENCODED '1 or 1=1'\n\n$ curl http://{LOAD_BALANCER_IP}/{APPLICATION_PATH}/api\\?id\\=1%20or%201%3D1\n\n\n{\"Note\":\"[(1, 'cat'), (2, 'dog'), (3, 'frog'), (4, 'hog'), (5, 'meow'), (6,\n'bark'), (7, 'ribbit'), (8, 'grunt')]\"}\n\n```\n\n\nThis shows us that we can exploit the SQL injection since we exposed data we\nshould not have access to.\n\nExploiting a vulnerability is not always as simple, so it is important to\ncombine resources as noted above\n\nto figure out exploitability.\n\n\n8) Determine a fix.\n\n\nNow that we know this is a problem within our system, we can use the\nprovided information to create an merge request (MR) to resolve\n\nand then test the MR in a non-production environment. Reviewing the\nvulnerability insights and AI response, we know we can solve this\n\nin a variety of ways. For example, we can:\n  \n* Use parameterized queries rather than directly calling the query\n\n* Sanitize the input before passing it to the `execute()` method\n\n\nTo enhance our knowledge, we should read\n[CWE-89](https://cwe.mitre.org/data/definitions/89.html) provided in the\nIdentifiers.\n\n\n9) Open the [GitLab\nWebIDE](https://docs.gitlab.com/ee/user/project/web_ide/) or editor of your\nchoice.\n\n\n10) Open the vulnerable file and scroll to the affected line of code. We\nfound this using the information provided in the insights.\n\n\n11) Apply the suggested change by reviewing the vulnerability insights and\nAI response. I changed the following:\n\n\n```python\n\ntry:\n  query = \"SELECT id, data FROM notes WHERE (secret IS FALSE AND id = %s)\" % id\n  if admin:\n    query =\"SELECT id, data, secret FROM notes WHERE (id = %s)\" % id\n  # NOT USING A PARAMETERIZED QUERY - SQL INJECTION CAN BE PASSED IN (,id)\n  cur.execute(query)\nexcept Exception as e:\n  note.logger.error(\"Error: cannot select note by id - %s\" % e)\n```\n\n\nto \n\n\n```python\n\ntry:\n  query = \"SELECT id, data FROM notes WHERE (secret IS FALSE AND id = %s)\"\n  if admin:\n    query =\"SELECT id, data, secret FROM notes WHERE (id = %s)\"\n  # USING A PARAMETERIZED QUERY - SQL INJECTION CANNOT BE PASSED IN (,id)\n  cur.execute(query, (id,))\nexcept Exception as e:\n  note.logger.error(\"Error: cannot select note by id - %s\" % e)\n```\n\n\nWe know this is the solution because parameterized queries as explained do\nnot allow actual SQL\n\ncommands to be run. Therefore, a SQL injection cannot be passed as the `id`.\nAdding a parameterized\n\nquery is easy since it is built into the Python db library we are using.\n\n\nThere may be multiple solutions to a vulnerability. It is up to the user to\ndecide what is best\n\nfor their application and workflow. The AI response provides a typical\nsolution, but more can be\n\nexamined and applied. For example, the AI response said we can add the\nfollowing:\n\n\n```python\n\ncur.execute(query.replace(\"'\", \"''\"))\n\n```\n\n\nThis would escape the single quotes in the input, making it safe to pass to\nthe `execute()` method.\n\nIt is a valid solution with less code required. However, I wanted to\nrestructure my code, so I applied\n\nanother solution found in the vulnerability insights.\n\n\n12) Create an MR with the fix. In my environment, feature branches are\nautomatically deployed\n\nto a new environment independent from production so we can test our features\nbefore merging them\n\nto production.\n\n\n13) Test the change in a non-production environment.\n\n\nOnce we push the MR, we can see if the vulnerability has been resolved and\nwe can test in a non-production\n\nenvironment:\n\n\n```bash\n\n# A REGULAR API-CALL\n\n$ curl http://{LOAD_BALANCER_IP}/{NEW_BRANCH_FIXED_APPLICATION_PATH}/api\n\n\n{\"Note\":\"[(1, 'cat'), (2, 'dog'), (3, 'frog'), (4, 'hog')]\"}\n\n\n# API CALL PASSING '1 or 1=1' AS SHOWN IN AI RESPONSE AND DETAILED IN\nIDENTIFIERS\n\n# NOTE: `1%20or%201%3D1` IS URL ENCODED '1 or 1=1'\n\n$ curl\nhttp://{LOAD_BALANCER_IP}/{NEW_BRANCH_FIXED_APPLICATION_PATH}/api\\?id\\=1%20or%201%3D1\n\n\n{\"Note\":\"[(1, 'cat')]\"}\n\n```\n\n\nWe can see that now the additional query parameters `or 1=1` are ignored and\nonly the first element\n\nis returned, meaning only the `1` was passed. We can further test if we can\nget item `5` which we should\n\nnot have access to:\n\n\n```bash\n\n# API CALL PASSING '5 or 1=1' AS SHOWN IN AI RESPONSE AND DETAILED IN\nIDENTIFIERS\n\n# NOTE: `5%20or%201%3D1` IS URL ENCODED '5 or 1=1'\n\n$ curl\nhttp://{LOAD_BALANCER_IP}/{NEW_BRANCH_FIXED_APPLICATION_PATH}/api\\?id\\=5%20or%201%3D1\n\n{\"Note\":\"[]\"}\n\n```\n\n\nSuccess, the SQL injection is no longer present!\n\n\n14) Merge into production.\n\n\nNow that we know the vulnerability has been resolved we can go ahead and\nmerge our fix! This is how you can use vulnerability insights\n\nto help resolve your vulnerabilities. If you wish to test all this for\nyourself, check out the complete [GitLab DevSecOps\ntutorial](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/).\n\n\n## Additional GitLab AI features\n\nAs we have seen above, Explain this Vulnerability assists you in remediating\nthe vulnerabilities within your\n\ndefault branch, but that's not the only AI feature GitLab has available!\nOther AI features to enhance your productivity include:\n\n\n* [Code\nSuggestions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions.html):\nEnables you to write code more efficiently by viewing code suggestions as\nyou type\n\n* [Suggested\nReviewers](https://docs.gitlab.com/ee/user/project/merge_requests/reviews/#suggested-reviewers):\nHelps you receive faster and higher-quality reviews by automatically finding\nthe right people to review a merge request\n\n* [Value Stream\nForecasting](https://docs.gitlab.com/ee/user/analytics/value_streams_dashboard.html):\nPredicts productivity metrics and identifies anomalies across your software\ndevelopment lifecycle\n\n* [Summarize Issue\nComments](https://docs.gitlab.com/ee/user/ai_features.html#summarize-issue-discussions):\nQuickly gets everyone up to speed on lengthy conversations to ensure you are\nall on the same page\n\n* [Summarize Proposed Merge Request\nChanges](https://docs.gitlab.com/ee/user/ai_features.html#summarize-my-merge-request-review):\nHelps merge request authors drive alignment and action by efficiently\ncommunicating the impact of their changes\n\n* [Summarize Merge Request\nReview](https://docs.gitlab.com/ee/user/ai_features.html#summarize-merge-request-changes):\nEnables better handoffs between authors and reviewers and helps reviewers\nefficiently understand merge request suggestions\n\n* [Generate Tests in Merge\nRequests](https://docs.gitlab.com/ee/user/ai_features.html#generate-suggested-tests-in-merge-requests):\nAutomates repetitive tasks and helps you catch bugs early\n\n* [GitLab\nChat](https://docs.gitlab.com/ee/user/ai_features.html#gitlab-duo-chat):\nHelps you quickly identify useful information in large volumes of text, such\nas documentation\n\n* [Explain this\nCode](https://docs.gitlab.com/ee/user/ai_features.html#explain-selected-code-in-the-web-ui):\nAllows you to get up to speed quickly by explaining source code\n\n\nVisit our [GitLab Duo site](https://about.gitlab.com/gitlab-duo/) to learn\nmore about these features, GitLab's mission around AI, and our partnership\nwith Google.\n",[720,830,9],{"slug":3744,"featured":6,"template":700},"remediating-vulnerabilities-with-insights-and-ai","content:en-us:blog:remediating-vulnerabilities-with-insights-and-ai.yml","Remediating Vulnerabilities With Insights And Ai","en-us/blog/remediating-vulnerabilities-with-insights-and-ai.yml","en-us/blog/remediating-vulnerabilities-with-insights-and-ai",{"_path":3750,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3751,"content":3757,"config":3762,"_id":3764,"_type":13,"title":3765,"_source":15,"_file":3766,"_stem":3767,"_extension":18},"/en-us/blog/remote-development-beta",{"title":3752,"description":3753,"ogTitle":3752,"ogDescription":3753,"noIndex":6,"ogImage":3754,"ogUrl":3755,"ogSiteName":685,"ogType":686,"canonicalUrls":3755,"schema":3756},"Behind the scenes of the Remote Development Beta release","Discover the epic journey of GitLab's Remote Development team as they navigate last-minute pivots, adapt, and deliver new features for users worldwide.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679888/Blog/Hero%20Images/remotedevelopment.jpg","https://about.gitlab.com/blog/remote-development-beta","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Behind the scenes of the Remote Development Beta release\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"David O'Regan\"}],\n        \"datePublished\": \"2023-08-16\",\n      }",{"title":3752,"description":3753,"authors":3758,"heroImage":3754,"date":3759,"body":3760,"category":785,"tags":3761},[2383],"2023-08-16","\nIn May 2023, the Create:IDE team faced an epic challenge – to merge the [Remote Development Rails monolith integration branch](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105783) into the `master` branch of the GitLab Project. This was no small ask, as the merge request was of considerable size and complexity. In this blog post, we'll delve into the background, justifications, and process behind this endeavor.\n\nThe merge request titled \"Remote Development feature behind a feature flag\" was initiated by the Create:IDE team, aiming to merge the branch \"remote_dev\" into the \"master\" branch in the Rails monolith GitLab project. The MR contained `4` commits, `258` pipelines, and `143` changes that amounted to a total of `+7243` lines of code added to the codebase.\n\nInitially, the MR was created to reflect the work related to \"Remote Development\" under the \"Category: Remote Development.\" It was primarily intended to have CI pipeline coverage for the integration branch and was not meant for individual review or direct merging. The plan was to merge this code into the master branch via the [\"Remote Development Beta - Review and merge\" Epic](https://gitlab.com/groups/gitlab-org/-/epics/10258).\n\n![SUM](https://about.gitlab.com/images/blogimages/remote-development/SUM.png){: .shadow.medium}\n\n### How the Remote Development project started\nAs a team, we embarked on an ambitious journey to create a greenfield feature: the [Remote Development](https://docs.gitlab.com/ee/user/project/remote_development/) offering at GitLab. This feature had a vast scope, many unknowns, and required solving numerous new problems. To efficiently tackle this task, we decided to work on an integration branch using a [low-ceremony process](https://stackoverflow.com/questions/68092498/what-does-low-ceremony-mean). This decision enabled us to develop and release the feature in an impressively short time frame of less than four months.\n\nWorking on an integration branch provided us the flexibility to make significant progress, but it was always intended to eventually break down the work into smaller, iterative MRs that would follow the standard [GitLab review process](https://docs.gitlab.com/ee/development/code_review.html). We had a [detailed plan](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/integration-branch-process.md#master-mr-process-summary) for this process, but we realized that following the original plan would not allow us to meet our goal of releasing of the feature in GitLab 16.0.\n\n### Merging the integration branch MR without breaking it up\nDuring the development of the Remote Development feature, our team faced several challenges that led us to adopt a new approach for merging the integration branch into the master. First, as part of our [velocity-based XP/Scrum style process](https://about.gitlab.com/handbook/engineering/development/dev/create/ide/#-remote-development-iteration-planning), we realized that meeting the 16.0 release goal would require us to cut scope. A velocity report, \"[Velocity-based agile planning report](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118436),\" highlighted that breaking down and reviewing individual MRs would take too long, considering the impending due date and the likelihood of last-minute scope additions.\n\nSecond, we [made the decision](https://gitlab.com/gitlab-org/gitlab/-/issues/398227#note_1361192858) to release workspaces as a **beta feature for public projects** for customers in [GitLab 16.0](/releases/2023/05/22/gitlab-16-0-released/#remote-development-workspaces-available-in-beta-for-public-projects). This approach reduced the complexity of the rollout plan and allowed us to get valuable feedback earlier, but required us to enable the feature by default earlier than planned. To align with this decision, we determined that merging the integration branch after review was the best course of action. An announcement was made to explain the change in plan, and we set specific timelines for the review process to ensure smooth coordination.\n\n> Hello Reviewers/Maintainers 👋 We have opened up a Zoom room through all of next week as an easy sync place for us all to collaborate and triage questions. As the MR is quite large, it might be overwhelming to determine where to begin. To help, we will aim to furnish a summary of what we have included, such as two new database tables and a couple of GraphQL/REST APIs. We will also be available through the week in the Zoom room and without it being too prescriptive of a approach, I would suggest we do a sync walkthrough of the MR first and then kick off the reviews.\n\nAddressing the concerns about risk, team members discussed the challenges and potential solutions. While there were apprehensions, we were confident in the overall quality of the feature. A disciplined plan for merging MRs was initially considered, but based on our velocity metrics, it was evident that meeting the public beta release goal required a new strategy.\n\nDespite the deviations from our usual practices, we acknowledged the urgency to deliver the initial release on time. The decision was not taken lightly, and we ensured that the merge had extensive [test coverage](https://docs.gitlab.com/ee/ci/testing/test_coverage_visualization.html) and [feature flags](https://docs.gitlab.com/ee/operations/feature_flags.html) in place to address any potential issues. We accepted that some aspects would be overlooked in the initial MR review cycle, but we committed to addressing them in subsequent iterations.\n\n### Keeping the pipeline green and stable for the merge\nTo ensure the successful merge of the integration branch containing the Remote Development feature, our team made significant efforts to keep the pipeline green and stable. As the MR was quite large and contained critical functionality, it was crucial to maintain a high level of quality and reduce the risk of introducing regressions.\n\nTo address these challenges, the team adopted a disciplined approach to [CI/CD](https://about.gitlab.com/topics/ci-cd/). Throughout the development process, CI pipelines were carefully monitored, and any failing tests or issues were promptly addressed. The team conducted rigorous testing and code reviews to identify and fix potential bugs and ensure that the changes did not negatively impact the existing functionality of the codebase.\n\nAdditionally, extensive test coverage was put in place to ensure that the new feature worked as expected and did not cause unintended side effects. The team utilized GitLab's [test coverage visualization](https://docs.gitlab.com/ee/ci/testing/test_coverage_visualization.html) capabilities to track the extent of test coverage and identify areas that required additional testing.\n\n![PIPE](https://about.gitlab.com/images/blogimages/remote-development/PIPE.png){: .shadow.medium}\n\n## The merging process\nAs part of the Remote Development team, we took a strategic approach to the merging process. We identified three categories of follow-up tasks that needed to be addressed after the release:\n\n1. **To-dos:** This category encompassed follow-up issues that required further attention.\n2. **Disabled linting rules:** Any issues related to disabled linting rules were included in this category.\n3. **Follow-up from review:** Non-blocking concerns raised during the review process were categorized here.\n\nTo manage this process effectively, we organized these categories into [child epics](https://docs.gitlab.com/ee/user/group/epics/manage_epics.html#multi-level-child-epics) under the main epic representing the merging effort.\n\n1. Child epic for [to-do follow-up issues](https://gitlab.com/groups/gitlab-org/-/epics/10472)\n2. Child epic for [disabled linting rules follow-up issues](https://gitlab.com/groups/gitlab-org/-/epics/10473)\n3. Child epic for [follow-up issues from review](https://gitlab.com/groups/gitlab-org/-/epics/10474)\n\n\n## Reviewer resources\nDuring the integration branch merge process for the Remote Development feature, we ensured a smooth and collaborative review experience for all involved. To facilitate this, we set up the following resources and documented the information in GitLab's issue, epic, and MR reviews for better persistence and traceability:\n\n1. **Dedicated Slack channel:** We had a Slack channel that served as our primary hub for coordinating reviews and resolving any blockers that arose during the process. The discussions, decisions, and important points discussed in this channel were documented in the related GitLab issues and epics. This approach enabled us to maintain a historical record of the conversations for to refer back to in the future.\n2. **General Slack channel:** For non-urgent or non-blocking questions and discussions, reviewers could use the a general Slack channel. Similar to the dedicated channel, we documented the relevant information from this channel in the corresponding issues and MR reviews in GitLab.\n3. **Addressing urgent issues:** When urgent issues required immediate attention, reviewers could directly address our technical leads [Vishal Tak](https://gitlab.com/vtak) and/or [Chad Woolley](https://gitlab.com/cwoolley-gitlab) in their Slack messages. However, we kindly requested that [direct messages were avoided](https://handbook.gitlab.com/handbook/communication/#avoid-direct-messages) to promote open collaboration. The resolutions to these urgent issues were documented in the corresponding GitLab issues or MR discussions.\n4. **Zoom collaboration room:** The collaborative sessions held in the open Zoom room were not only beneficial for real-time discussions but also for fostering a collaborative environment. After each session, we summarized the key points and decisions made during the meeting in the associated GitLab issue or MR, making sure all important outcomes were captured and accessible to the team.\n\nThroughout the review process, we were committed to maintaining a seamless and well-documented workflow. By capturing all relevant information in GitLab issues, epics, and MR reviews, we ensured that the knowledge was persistently available, and future team members could easily understand the context and decisions made during the integration process.\n\n## Application security review\nDuring the application security review process, we focused on providing a secure and reliable Remote Development feature for our users. Here are the key resources and updates related to the application security review:\n\n1. **Main application security review issue:** The main application security review issue served as the central hub for tracking security-related considerations. You can find the defined process we followed [here](https://about.gitlab.com/handbook/security/security-engineering/application-security/appsec-reviews.html).\n2. **Application security review comment:** The application security review issue contained a comment indicating that the merge was not blocked unless there were severe issues that could impact production. \"In order to maintain a smooth merge process, we do not block MRs from being merged unless we identify severe issues that could prevent the feature from going into production, such as S1 or S2 level problems. If you are aware of any design flaws or concerns that might qualify as such issues, please bring them to our attention. We can review them together and address any questions or concerns that arise. Let's work collaboratively to find an approach that works for both parties. 👍\"\n3. **Engineering perspective:** For managing the application security review process from an engineering team perspective, we had a dedicated issue, which is kept confidential for security reasons.\n4. **Security and authentication matters:** All security and authentication concerns pertaining to the Beta release were documented within the [`Remote Development Beta -Auth` epic](https://gitlab.com/groups/gitlab-org/-/epics/10377). As of April 30, 2023, we are delighted to announce that **no known issues or obstacles were found that would impede the merge**. This represents a significant accomplishment, considering the intricate nature of this new feature.\n5. **Initial question raised:** During the application security review, one initial question was raised, and we promptly addressed it. You can track the issue and our response [here](https://gitlab.com/gitlab-org/gitlab/-/issues/409317).\n\n## Database review\nTo ensure the reliability and efficiency of the Remote Development feature, we sought guidance from the database reviewer. Although the team had not conducted a thorough self-review, we were fully prepared to address any blocking issues raised during the review process. Our references for the review were:\n\n- [Database review documentation](https://docs.gitlab.com/ee/development/database_review.html)\n- [Database reviewer guidelines](https://docs.gitlab.com/ee/development/database/database_reviewer_guidelines.html)\n\nAs an example, during the database migration review, a discussion arose between [Alper Akgun](https://gitlab.com/a_akgun) and Chad, regarding the efficient ordering of columns in the workspaces table. Alper initially suggested placing integer values at the beginning of the table based on relevant documentation.\n\nChad questioned the benefit of this suggestion, pointing out that the specific integer field, `max_hours_before_termination`, would still be padded with empty bytes even if moved to the front, due to its current position between two text fields.\n\nAlper proposed an alternative approach, emphasizing that organizing variable-sized fields (such as `text`, `varchar`, `arrays`, `json`, `jsonb`) at the end of the table could be sufficient for the workspaces table.\n\nUltimately, Chad took the initiative to implement the changes, moving all variable length fields to the end of the table, and documented the discussion as a comment to address review suggestions.\n\nWith this collaborative effort, the workspaces table was efficiently optimized, and the team gained valuable insights into database column ordering strategies.\n\n![DB](https://about.gitlab.com/images/blogimages/remote-development/DB.png){: .shadow.medium}\n\n## Ruby code review\nDuring the Ruby code review phase, we followed a meticulous approach by conducting a comprehensive self-review of every line of code. Our goal was to ensure the highest code quality and address any potential issues identified by the reviewers effectively.\n\nTo ensure clarity, it's important to clarify that the Ruby code review primarily focused on backend changes and server-side improvements. This included optimizing performance, enhancing functionalities, and refining the overall codebase to deliver a seamless user experience.\n\nFor the code review process, we referred to the [Code review documentation](https://docs.gitlab.com/ee/development/code_review.html), a valuable resource that guided us in maintaining industry best practices and adhering to the GitLab community's coding standards.\n\n### Example: Enhance error messages for unavailable features\nAs an example during the code review, we addressed an essential aspect of the workspace method, focusing on how we handle scenarios related to the `remote_development_feature_flag` and the `remote_development` licensed feature. The primary objective was to enhance the error messages presented to users when these features are not available.\n\nInitially, the code employed identical error messages for both cases, making it less clear to users whether the issue was due to a missing license or a disabled feature flag. This ambiguity could lead to confusion and hinder the user experience.\n\n#### The suggested improvement\nDuring the review, one of our maintainers, [Peter Leitzen](https://gitlab.com/splattael), raised an important question: \"Are we OK with having only a single error message for both cases (missing license and missing feature flag)?\"\n\nRecognizing the importance of clear communication, Chad proposed enhancing the error messages to provide distinct descriptions for each case. This improvement aimed to empower users by precisely conveying the reason behind the unavailability of certain features.\n\n#### The revised implementation\nFollowing Chad's suggestion, the code underwent the following changes:\n\n```ruby\nunless ::Feature.enabled?(:remote_development_feature_flag)\n  # TODO: Could have `included Gitlab::Graphql::Authorize::AuthorizeResource` and then use\n  #       raise_resource_not_available_error!, but didn't want to take the risk to mix that into\n  #       the root query type\n  raise ::Gitlab::Graphql::Errors::ResourceNotAvailable,\n    \"'remote_development_feature_flag' feature flag is disabled\"\nend\n\nunless License.feature_available?(:remote_development)\n  # TODO: Could have `included Gitlab::Graphql::Authorize::AuthorizeResource` and then use\n  #       raise_resource_not_available_error!, but didn't want to take the risk to mix that into\n  #       the root query type\n  raise ::Gitlab::Graphql::Errors::ResourceNotAvailable,\n    \"'remote_development' licensed feature is not available\"\nend\n\nraise_resource_not_available_error!('Feature is not available') unless current_user&.can?(:read_workspace)\n```\n\n#### The value of distinct error messages\nBy implementing distinct and descriptive error messages, we reinforce our commitment to user-centric development. Users interacting with our system will receive accurate feedback, helping them navigate potential roadblocks effectively. This enhancement not only improves the user experience but also streamlines troubleshooting and support processes.\n\nThis code review example highlights the significance of concise and informative error messages in delivering a top-notch user experience within the GitLab ecosystem. Our team's collaborative efforts ensure that users can confidently interact with our platform, knowing they'll receive clear and helpful error messages when needed.\n\n![BE1](https://about.gitlab.com/images/blogimages/remote-development/BE1.png){: .shadow.medium}\n\n### Example: Improving performance and addressing N+1 issues in WorkspaceType\nIn a recent code review, our team focused on optimizing the WorkspaceType and addressing potential N+1 query problems. The discussion involved two key contributors, [Laura Montemayor](https://gitlab.com/lauraX) and Chad, who worked together to enhance the performance of the codebase.\n\n#### Identifying the performance concerns\nDuring the review, Laura raised a performance concern regarding the possibility of N+1 queries in the WorkspaceType resolver. She suggested that preloading certain associations could be beneficial to avoid this common performance issue.\n\n#### A separate issue for N+1 control\nChad took prompt action and created a separate issue specifically aimed at resolving the N+1 query problems. The new issue, titled \"Address review feedback: Resolve N+1 issues,\" would address the concerns raised by Laura and implement the necessary preloading.\n\n#### Evaluating the potential N+1 impact\nChad provided insightful information about the low risk of real N+1 impact from two particular fields in the current implementation. He elaborated on how the queries for user and agent associations would largely be cache hits due to scoping and usage patterns. Chad diligently examined the cache hits happening in development, confirming the potential optimization.\n\nHere's a code snippet from the initial implementation:\n\n```ruby\n# Initial Implementation\nclass WorkspaceType \u003C BaseType\n  field :user, ::Types::UserType,\n    description: \"User associated with this workspace\",\n    null: true\n\n  field :agent, ::Types::AgentType,\n    description: \"Agent associated with this workspace\",\n    null: true\n\n  # Resolver for the user association\n  def user\n    object.user\n  end\n\n  # Resolver for the agent association\n  def agent\n    object.agent\n  end\nend\n```\n\n#### Treating performance as a priority\nBoth contributors acknowledged the significance of addressing the performance concern, with Laura emphasizing its importance. They agreed to prioritize the separate issue dedicated to resolving the N+1 queries and ensuring proper test coverage.\n\nHere's a code snippet from the revised implementation:\n\n```ruby\n# Revised Implementation with Preloading\nclass WorkspaceType \u003C BaseType\n  field :user, ::Types::UserType,\n    description: \"User associated with this workspace\",\n    null: true\n\n  field :agent, ::Types::AgentType,\n    description: \"Agent associated with this workspace\",\n    null: true\n\n  # Resolver for the user association with preloading\n  def user\n    ::Dataloader.for(::User).load(object.user_id)\n  end\n\n  # Resolver for the agent association with preloading\n  def agent\n    ::Dataloader.for(::Agent).load(object.agent_id)\n  end\nend\n```\n\n#### Considering future usage\nChad expressed excitement about the possibility of the new feature gaining significant usage. He humorously stated that encountering enough legitimate traffic on workspaces to trigger any performance impact would be a delightful problem to have, as it would indicate a growing user base.\n\n#### Collaboration and performance improvement\nThe code review exemplifies the collaborative and proactive approach of our team in optimizing the WorkspaceType. The team's dedication to addressing performance concerns ensures that our codebase remains performant and efficient, even as our user base grows.\n\n![BE2](https://about.gitlab.com/images/blogimages/remote-development/BE2.png){: .shadow.medium}\n\n## Frontend code review\nThe frontend code review process was managed by our resident `Create: IDE` frontend maintainers, [Paul Slaughter](https://gitlab.com/pslaughter) and [Enrique Alcátara](https://gitlab.com/ealcantara). Additionally, a significant portion of the new frontend UI code had already undergone separate reviews and was merged to master, contributing to the overall quality of the Remote Development feature.\n\n### Example: Collaborative code improvement for ApolloCache Mutators\nPaul started a thread on an old version of the diff related to `ee/spec/frontend/remote_development/pages/create_spec.js``. The code snippet in question involved creating a mock Apollo instance and writing queries to the cache.\n\n#### The initial implementation\nInitially, the code involved writing to the cache twice, which raised concerns among the maintainers, Paul and Enrique. Paul pointed out that the duplicate write was unintentional and wondered if the writeQuery was even necessary, given the removal of @client directives. However, he also acknowledged the need to test that the created workspace was added to the ApolloCache.\n\n```javascript\n// Initial Implementation\nconst buildMockApollo = () => {\n  // ... Other mock setup ...\n\n  // Initial writeQuery for userWorkspacesQuery\n  mockApollo.clients.defaultClient.cache.writeQuery({\n    query: userWorkspacesQuery,\n    data: USER_WORKSPACES_QUERY_EMPTY_RESULT.data,\n  });\n\n  // ... Other mock setup ...\n};\n```\n\n#### Identifying a potential issue\nEnrique agreed that the duplicate write was unintentional and probably introduced during a rebase. He explained that pre-populating the cache with a user workspaces query empty result was essential for the mutator to have a place to add the workspace. However, he encountered difficulties in making the workaround work effectively in unit tests.\n\n#### Resolving the issue\nPaul highlighted the significance of pre-populating the cache with the user workspaces query empty result. He suggested leaving a comment to explain the necessity of the initial writeQuery, as it would be implicitly coupled to future writeQuery operations.\n\n```javascript\n// Resolving the Issue - Leaving a Comment\n// Pre-populate the cache with user workspaces query empty result to provide a place\n// for the mutator to add the Workspace later. This is needed for both test and production environments.\nmockApollo.clients.defaultClient.cache.writeQuery({\n  query: userWorkspacesQuery,\n  data: USER_WORKSPACES_QUERY_EMPTY_RESULT.data,\n});\n```\n\nHowever, upon further investigation, Paul discovered that the writeQuery might not be needed, and the issue might be a symptom of an underlying problem. He decided to open a separate thread to address this concern and indicated that he would work on a separate MR to handle it.\n\n```javascript\n// Resolving the Issue - Opening a Separate Thread and MR\n// Open a separate thread to discuss potential underlying issues.\n// Plan to work on a separate MR to handle it.\n// Stay tuned for updates!\n```\n\n![FE](https://about.gitlab.com/images/blogimages/remote-development/FE.png){: .shadow.medium}\n\n## What we learned\nAs part of the Remote Development team, we faced the challenge of merging the Remote Development Rails monolith integration branch to meet our ambitious release goal. We adapted to last-minute pivots and focused on minimizing risks during the review process. The successful merge brought us one step closer to benefiting GitLab users worldwide. We acknowledged areas for improvement and remained committed to refining the feature's quality. Our journey reflects our dedication to delivering results, embracing change, and pushing boundaries in the DevOps community. The release of the Remote Development feature in GitLab 16.0 is a significant milestone for GitLab, and we continue to iterate and grow, providing innovative solutions for developers worldwide.\n\nAn outcome of this process was an ongoing conversation to propose a [simplified review process for greenfield features](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125117). Through this proposal, we aim to distill the lessons we learned during this experience and provide guidance to future teams facing similar challenges.\n\n## What is next for Remote Development?\nAfter the merge of the MR, several changes were implemented:\n- The first production tests were conducted to ensure the stability and functionality of the merged code.\n- Collaboration took place between the Dev Evangelism and Technical Marketing teams, focusing on [creating content](https://gitlab.com/groups/gitlab-com/marketing/developer-relations/-/epics/190). This collaboration aimed to troubleshoot any issues that arose during the merge.\n- Feedback from the community was taken into account, and changes were made to address the concerns raised. This feedback was incorporated into an [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/410031) and influenced the overall roadmap and direction of the project.\n\nDo you want to [contribute to GitLab](/community/contribute/)? Come and join in the conversation in the `#contribute` channel on GitLab's [Discord](https://discord.gg/gitlab), or just pop in and say \"Hi.\"\n\n",[9,851,956,1898,1306,743],{"slug":3763,"featured":6,"template":700},"remote-development-beta","content:en-us:blog:remote-development-beta.yml","Remote Development Beta","en-us/blog/remote-development-beta.yml","en-us/blog/remote-development-beta",{"_path":3769,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3770,"content":3775,"config":3782,"_id":3784,"_type":13,"title":3785,"_source":15,"_file":3786,"_stem":3787,"_extension":18},"/en-us/blog/rust-programming-language",{"title":3771,"description":3772,"ogTitle":3771,"ogDescription":3772,"noIndex":6,"ogImage":2652,"ogUrl":3773,"ogSiteName":685,"ogType":686,"canonicalUrls":3773,"schema":3774},"A guide to Rust programming language","Rust is a well-loved programming language but it is a mindset shift from options like C++. Here's a tutorial and an inside look at Rust code and its capabilities.","https://about.gitlab.com/blog/rust-programming-language","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"A guide to Rust programming language\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Valerie Silverthorne\"}],\n        \"datePublished\": \"2020-07-21\",\n      }",{"title":3771,"description":3772,"authors":3776,"heroImage":2652,"date":3778,"body":3779,"category":3780,"tags":3781},[3777],"Valerie Silverthorne","2020-07-21","\n\n## What is Rust?\n\nRust is an open source programming language that has been the \"most loved language\" on developer community Stack Overflow's annual survey for the last four years. While it's a popular language in that sense, only a very, _very_ small number of developers actually use Rust language today – a July 2020 look at the PYPL PopularitY of Programming Languages Index ranks it at number 18 with just .81% interest. (For comparison Python is at nearly 32% and Java is over 17%.)\n\nSo why the intense love of the Rust programming language? To put it simply, Rust coding was created to solve problems present in other languages and if you can take the time to unlock its (admittedly difficult) secrets, you're rewarded with cleaner, faster, and most importantly, safer code. Rust code resolves pain points that you see in countless other programming languages with far fewer downsides. Utilizing Rust allows developers to decide when they no longer need memory at the time of compilation which creates more efficiency around memory usage.\n\n[Antony Saba](/company/team/#asaba), a senior security engineer with Strategic Security at GitLab, recently talked about Rust during a company-wide series of meetings ([Contribute 2020](/events/gitlab-contribute/)). He speaks from experience as his last employer was a Rust-based company. \"Okay, so what's Rust's promise?\" Saba asked. \"Rust's promise is that it should be easier, and everybody should be able to fearlessly write at a systems level and not have to worry about memory safety or thread safety, or at least worry about it in the way that is supported by the language and the tools.\"\n\nLet's unpack what that means.\n\n## History of Rust programming language\n\nThe [open source Rust community](https://www.rust-lang.org) describes the language as fast, reliable and productive. \"Hundreds of companies around the world are using Rust in production for fast, low-resource cross-platform solutions,\" the organization says. Firefox and DropBox are two well-known users of Rust today, and Mozilla (creator of Firefox) was the first original supporter of Rust.\n\n### Who created Rust?\n\nRust code was originally developed as an open source project by software developer Graydon Hoare while working at Mozilla Research in 2006 and has been maintained by the Rust Foundation since 2021. It’s now one of the top drivers of the Rust programming language. \n\nThink of Rust as the answer to a data-rich problem that will likely need lots of computational cycles. Mozilla's [Rust documentation](https://research.mozilla.org/rust/) specifically calls out the language as ideal for \"game engines, operating systems, file systems, browser components and simulation engines for virtual reality.\"\n\n## Benefits of programming in Rust\n\nThe top benefit of Rust coding is its adept memory management. Although there are other programming languages that emphasize memory safety like Rust code, Rust handles the concept differently in that it doesn’t use a garbage collector as other programming languages do. Instead, Rust uses a borrow checker to track variable scope and object lifetime while simultaneously administering high-quality memory safety and stopping concurrent data races. \n\nThe benefits of programing in Rust don’t stop at memory management. It’s fast and reliable for creating web apps and creating cross-platform applications, and it can integrate with preexisting code. \n\nOne of the other major benefits of Rust programming language is that it is well-suited for projects that demand extremely high performance. Its ability to process large amounts of data and CPU-intensive operations makes it a strong competitor in the developer space. \n\nOther Rust feature benefits offer a list of features that makes it stand out from other programming languages. Here are some of the features:\n\n1. It’s more user-friendly.\n2. You’ll find high-quality documentation about the language.\n3. It has a better resolution of memory errors and concurrent programs than C and C++ languages.\n4. It’s incredibly fast and highly secure compared to other languages.\n\n## The Rust ecosystem\n\nThe [JetBrains 2021 Developer Ecosystem Report](https://www.jetbrains.com/lp/devecosystem-2021/rust/) found that Rust developers have mostly been using it for less than six months, and often reach for the language for \"hobby\" or personal projects. What are devs primarily writing with Rust? The report found command line interface tools, systems programming and web development were the most popular options.\n\nMany companies have started using Rust, though. In 2020, [Discord switched from Go to Rust](https://discord.com/blog/why-discord-is-switching-from-go-to-rust), and Shopify, Dropbox, AWS and many others use it as well. \n\n## The basics of Rust programming language\n\nRust is a bit of a hybrid, according to Mozilla's Rust documentation. Rust offers developers the syntax advantages of high-level languages with the \"control and performance of a low-level language,\" the documentation explains.\n\nRust is a statically typed language rather than a dynamic one. Though developers like to argue the merits of both, Rust, like popular TypeScript, eliminates the frustration of \"dynamic typing.\" Data is constrained and checked by a compiler so confusion is minimized. Rust programming also makes it very hard to ignore errors – Steve Donovan, author of [\"A Gentle Guide to Rust,\"](https://stevedonovan.github.io/rust-gentle-intro/) jokes it can be hard not to think the compiler is shouting at you when you make a mistake.\n\nDonovan identifies Rust's key principles as:\n\n* Strictly enforcing safe borrowing of data\n* Functions, methods, and closures to operate on data\n* Tuples, structs, and enums to aggregate data\n* Pattern matching to select and destructure data\n* Traits to define behaviour on data\n\n## Types of Rust coding\n\nRust treats values by breaking them down into \"types\" in order to handle the data appropriately. According to MIT's guide to Rust, there are a number of types that can be split into scalar or integer types. Scalar types will likely be familiar to those who work with other programming languages: characters, Booleans, floating-point numbers and integers. They all represent a single value. Compound types are what they sound like – multiple types together.  \n\n## Who uses Rust?\n\nAll of the guardrails mentioned lead to a language that can create fast-moving code with few things that slow it down. There's no runtime or garbage collection, making coding in Rust ideal for applications where memory usage is at a premium (like embedded devices). But if there is a place where Rust really stands out, it's security. Donovan points out that Rust is \"safe by default,\" unlike C or C++. No one can corrupt memory by default, he writes.\n\n## The Rust programming language and productive coding\n\nAfter three years of coding in Rust, Antony was quick to say he's probably more productive with the language than any other.\n\n\"I really do feel like Rust was the most productive language I've ever used,\" he says. \"Once you are doing everything in that functional style, you're writing less code, but it's still clearer, because you don't have temporary variables. They're a thing that you don't really end up using when you're writing code that way. So, to me, it's those little things that I get the productivity out of.\"\n\n## Rust can be touchy, but rewarding\n\nProductive, sure, but there's a learning curve with Rust.\n\n\"It's true the borrow checker is the hardest part,\" Antony says. \"But the thing is, once you get past that, there is a serious dopamine hit when that program compiles, because it means now you only have your own logic errors to deal with. Part of that pain is explicitly some of the things that you assume, and some of the little white lies that you tell yourself when you're starting, especially with a C program. Because when you start your C program, it's like, 'all right, I have a couple command line parameters, I don't really want to write all my functions just to pass them, so I'm just going to declare a couple global variables and shove them in there, and I'll clean it up later.' Right? It's one of those little lies we tell ourselves. But you can't have immutable global variables in Rust. It just won't let you. You have to wrap it – you may as well just do the functions right. They're going to use your command-line arguments. It's the same with thread safety. You kind of have to do that upfront, and you don't get to make that assumption.\"\n\n## Looking to the future of the Rust programming language\n\nRust has a bright future, even if it might not be as widespread as other languages, Antony explains. \"I don't think it's ever going to be as popular as Go, just because Google is Google, and there's a lot of places that Go is really good for,\" he says. \"But for those places where you really want that fearless development, I think it'll continue to have a strong hold there.\"\n\nWatch Antony's Rust demo in full here:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/INT_rGJr6JQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n**Read more about programming languages:**\n\nCan we solve the [COBOL programmer shortage?](/blog/cobol-programmer-shortage/)\n\nWhy we use [Ruby on Rails](/blog/why-we-use-rails-to-build-gitlab/) to build GitLab\n\nHow [Modern C and C ++ work](/blog/conan-c-cpp-package-management-integration/)\n\nCover image by [Zsolt Palatinus](https://unsplash.com/@sunitalap) on [Unsplash](https://unsplash.com)\n{: .note}\n","insights",[743,830,9],{"slug":3783,"featured":6,"template":700},"rust-programming-language","content:en-us:blog:rust-programming-language.yml","Rust Programming Language","en-us/blog/rust-programming-language.yml","en-us/blog/rust-programming-language",{"_path":3789,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3790,"content":3796,"config":3801,"_id":3803,"_type":13,"title":3804,"_source":15,"_file":3805,"_stem":3806,"_extension":18},"/en-us/blog/safe-without-silos-in-gitlab",{"title":3791,"description":3792,"ogTitle":3791,"ogDescription":3792,"noIndex":6,"ogImage":3793,"ogUrl":3794,"ogSiteName":685,"ogType":686,"canonicalUrls":3794,"schema":3795},"SAFe without silos in GitLab","Learn how to map the Scaled Agile Framework to the native capabilities of the DevSecOps platform and the advantages that come from doing so.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097569/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2811%29_2hcwWx49wQ7CHfvhhkVH6S_1750097569126.png","https://about.gitlab.com/blog/safe-without-silos-in-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"SAFe without silos in GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Amanda Rueda\"}],\n        \"datePublished\": \"2025-04-08\",\n      }",{"title":3791,"description":3792,"authors":3797,"heroImage":3793,"date":3798,"body":3799,"category":2720,"tags":3800},[2717],"2025-04-08","Let's talk about what happens when your organization adopts the Scaled Agile Framework (SAFe) to scale to enterprise levels. You've got multiple teams working on complex products, and you need a way to coordinate all that work. But here's a common headache: Your planning happens in one tool, while your actual development work lives somewhere else entirely.\n\nThis divide creates real problems day-to-day. Developers jump between systems constantly. Product managers struggle to get an accurate picture of progress. And everyone wastes time manually copying information from one place to another. It's precisely the kind of disjointed experience that SAFe was designed to eliminate.\n\nWhile your development teams might already be using GitLab for source code management, CI/CD, and security, you may wonder whether GitLab can also support your planning needs within the SAFe framework. The good news is that GitLab's Agile project management capabilities offer strong support for SAFe, in this article, you'll learn how GitLab maps to SAFe concepts and ceremonies, all within the same DevSecOps platform your software developers already know and love.\n\n## What is SAFe?\n\nSAFe, or the Scaled Agile Framework, is a way to bring Agile principles to large organizations without losing speed, alignment, or customer focus. It takes the iterative and flexible teamwork model of small teams and applies its principles across big organizations that have multiple teams, roadmaps, and stakeholders. This brings the organization into alignment, all planning and executing in the same direction. For product managers, SAFe helps connect strategy to execution so you’re not just shipping fast, you’re shipping the right things, backed by clear priorities and cross-team alignment.\n\nSAFe reduces silos, encourages collaboration, and helps teams rally around customer outcomes, not just tasks. When integrated in GitLab, the magic really happens: visibility, traceability, and delivery all live in one place.\n\n## SAFe terminology in GitLab\n\nFirst, let's establish how SAFe concepts map to GitLab:\n\n| SAFe | GitLab |\n| :---- | :---- |\n| Epic | Top-level Epic |\n| Capability | Sub-epic (Level 1) |\n| Feature | Sub-epic (Level 2) |\n| User Story | Issue |\n| Task | Task |\n| Team | Custom Field / Scoped Label |\n| Sprint | Iteration |\n| Program Increment (PI) | Milestone |\n| Value Stream | Top-level Group |\n| Agile Release Train (ART) | Top-level Group |\n\n\u003Cbr>\u003C/br>\n\nWith this mapping as your guide, you can set up GitLab to mirror your SAFe implementation. The group structure lets you organize around your value streams and ARTs, while the work item hierarchy (with up to seven levels of nested epics!) gives you all the depth you need for complex product portfolios. Whether you're working at the portfolio level (with top-level groups), program level (with subgroups), or team level (with projects), GitLab's organizational structure aligns perfectly with SAFe's hierarchy.\n\n## Supporting SAFe ceremonies in GitLab\n\nNow for the fun part - how do you actually run your SAFe ceremonies in GitLab? Let's walk through each one.\n\n### PI planning\n\nTo facilitate the cross-team alignment and dependency management that makes PI planning successful, GitLab offers several capabilities:\n\n* Use the [Roadmap](https://docs.gitlab.com/user/group/roadmap/) view to visualize features across teams and time periods\n* Assign features to the PI [milestone](https://docs.gitlab.com/user/project/milestones/)\n* Document and visualize cross-team [dependencies](https://docs.gitlab.com/user/project/issues/related_issues/#blocking-issues) as they're identified\n\nGitLab gives you flexibility for PI planning through both the Epic boards (which can be configured to show team assignments) and the Roadmap view (which shows features over time like a Gantt chart). You can switch between these views during your planning session depending on whether you're focusing on the timeline or team organization.\n\n![Roadmap view and epic board](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097576746.gif)\n\n\u003Cbr>\u003C/br>\n\n![Roadmap view with Gantt chart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097576747.png)\n\n### Refinement\n\nAs a product manager, running effective refinement sessions means having clear visibility into your feature backlog. You can run your refinement session right inside GitLab. No more updating one tool during the meeting and then having to update another tool afterward.\n\nGitLab powers refinement sessions with:\n\n* [Epic boards](https://docs.gitlab.com/user/group/epics/epic_boards/) that group features based on status\n* The ability to view story points directly in the [overview](https://docs.gitlab.com/user/group/epics/epic_boards/#view-count-of-issues-weight-and-progress-of-an-epic)\n* Comprehensive [drawer views](https://docs.gitlab.com/user/group/epics/manage_epics/#open-epics-in-a-drawer) that let you interact with work items without losing context\n* The ability to create and link [child issues](https://docs.gitlab.com/user/group/epics/manage_epics/#add-an-issue-to-an-epic) directly from epics\n\n![SAFe - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097576749.gif)\n\n### Sprint planning\n\nWhen it's time to figure out what your team can tackle in the next sprint, GitLab gives you:\n\n* [Issue boards](https://docs.gitlab.com/user/project/issue_board/) that provide a comprehensive view of your backlog\n* [Total weight](https://docs.gitlab.com/user/project/issue_board/#sum-of-issue-weights) of user stories displayed directly on boards\n* The ability to easily move issues between iterations\n* A collapsible view that simplifies moving stories between sprints\n\nThis means you can keep everything in one place and spend your planning meetings actually planning instead of jumping between tools.\n\n![Sprint planning with GitLab](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097576751.gif)\n\n*💡 Check out [this tutorial on using GitLab to facilitate Scrum](https://docs.gitlab.com/tutorials/scrum_events/) for a detailed glimpse into the power of GitLab in Agile planning and sprint tracking.*\n\n### Daily stand-ups\n\nYour team can gather around the board during daily stand-ups and actually see what everyone's working on, what's stuck, and what's ready for review – all in one view. For your dev team's daily stand-ups, GitLab lets you:\n\n* Create [iteration-scoped](https://docs.gitlab.com/user/project/issue_board/#iteration-lists) boards that show the current sprint's work\n* Display story points/weights directly on cards\n* Use the [drawer view](https://docs.gitlab.com/user/project/issues/managing_issues/#open-issues-in-a-drawer) to access details without leaving the context\n* Highlight tasks at risk through [health status](https://docs.gitlab.com/user/project/issues/managing_issues/#health-status)\n\n![Daily stand-up board](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097576755.png)\n\n### Sprint review\n\nWant to know how your team is doing over time? GitLab provides comprehensive metrics with:\n\n* [Burndown and burnup charts](https://docs.gitlab.com/user/group/iterations/#iteration-burndown-and-burnup-charts) for iterations\n* Velocity tracking\n* [Lead and cycle time](https://docs.gitlab.com/user/group/value_stream_analytics/#lifecycle-metrics) metrics\n* Dashboards that can be scoped to teams\n\nThese metrics help you understand if your team is getting faster, where they're getting stuck, and what you might want to talk about in your next retrospective.\n\n![Burndown and burnup charts](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097577/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097576758.png)\n\n## 5 reasons a unified platform provides an advantage\n\nI know there are plenty of planning tools that can handle SAFe ceremonies. But there are game-changing reasons why I genuinely believe GitLab is different:\n\n1. **No more context switching** - Your planning, coding, testing, and security all happen in one place.\n2. **Everything's connected** - You can trace work from the big epic down to the code and deployment.\n3. **Everyone's on the same page** - Developers, product folks, and security teams all work together in the same tool.\n4. **Total visibility** - Stakeholders have one place to check for updates.\n5. **The full picture** - You see planning and development metrics together, so you know what's really going on.\n\nIf your dev teams already love GitLab, why make them jump to another tool for planning or create some complex, cobbled-together integrations? Bringing your SAFe planning into GitLab creates a much smoother experience for everyone.\n\n## Implementation principles\n\nI've worked with teams transitioning from traditional SAFe tools to GitLab, and here's what I've learned: Focus on **what each ceremony is trying to accomplish**, not on recreating exact replicas of your old tools.\n\nThe teams that get the most out of GitLab are the ones who embrace its native capabilities instead of fighting against them. Yes, it takes some initial work to figure out how to map your SAFe concepts and set up your workflows. But once you do, you'll find your processes actually get simpler rather than more complex.\n\nThe key is defining conventions that everyone follows. Which labels mean what? How will you track teams? What goes in an epic versus an issue? With a little upfront investment in these decisions, you'll end up with an intuitive system that eliminates all that cross-tool coordination overhead.\n\n## Getting started\n\nReady to give this a shot? Here's how to start implementing SAFe in GitLab:\n\n1. **Set up your structure** - Create groups and subgroups that [match your organization](https://about.gitlab.com/blog/best-practices-to-set-up-organizational-hierarchies-that-scale/).\n2. **Define your work breakdown** - Decide how you'll use [epics](https://about.gitlab.com/blog/unlocking-agile-excellence-gitlab-epics-for-seamless-portfolio-management/), [issues](https://docs.gitlab.com/user/project/issues/managing_issues/), and [tasks](https://docs.gitlab.com/user/tasks/).\n3. **Create your iterations** - Set up your [sprint schedule](https://docs.gitlab.com/user/group/iterations/#create-an-iteration-cadence).\n4. **Add your milestones** - [Milestones](https://docs.gitlab.com/user/project/milestones/#create-a-milestone) will represent your Program Increments in GitLab.\n5. **Build your boards** - Create different views for different ceremonies.\n6. **Agree on conventions** - Document how you'll use labels and custom fields.\n\nTaking time to think through these decisions upfront will save you many headaches later. And remember, you don't have to perfect it on day one - you can always adjust as you learn.\n\n## Bringing it all together\n\nGitLab gives you a solid foundation for running SAFe, especially if your dev teams are already GitLab fans. When you bring planning and development into the same tool, you eliminate those painful handoffs, make collaboration way easier, and get everything moving faster.\n\nThe beauty of GitLab's planning tools is that they're flexible enough to adapt to your specific flavor of SAFe. You're not locked into rigid workflows - you can evolve your approach as your teams mature and your needs change.\n\n> Ready to see how much better life is without those planning silos? [Start your free trial today](https://about.gitlab.com/free-trial/) and experience firsthand how GitLab can transform your SAFe implementation.\n\n*💡 If you liked this topic check out this related post - [GitLab for Agile Software Development](https://about.gitlab.com/blog/gitlab-for-agile-software-development/)*\n",[996,495,696,694,9],{"slug":3802,"featured":90,"template":700},"safe-without-silos-in-gitlab","content:en-us:blog:safe-without-silos-in-gitlab.yml","Safe Without Silos In Gitlab","en-us/blog/safe-without-silos-in-gitlab.yml","en-us/blog/safe-without-silos-in-gitlab",{"_path":3808,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3809,"content":3814,"config":3820,"_id":3822,"_type":13,"title":3823,"_source":15,"_file":3824,"_stem":3825,"_extension":18},"/en-us/blog/second-law-of-complexity-dynamics",{"title":3810,"description":3811,"ogTitle":3810,"ogDescription":3811,"noIndex":6,"ogImage":2238,"ogUrl":3812,"ogSiteName":685,"ogType":686,"canonicalUrls":3812,"schema":3813},"How pursuit of simplicity complicates container-based CI","Simplicity always has a certain player in mind - learn how to avoid antipatterns by ensuring simplicity themes do not compromise your productivity by over-focusing on machine efficiencies.","https://about.gitlab.com/blog/second-law-of-complexity-dynamics","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"When the pursuit of simplicity creates complexity in container-based CI pipelines\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Darwin Sanoy\"}],\n        \"datePublished\": \"2022-05-24\",\n      }",{"title":3815,"description":3811,"authors":3816,"heroImage":2238,"date":3817,"body":3818,"category":785,"tags":3819},"When the pursuit of simplicity creates complexity in container-based CI pipelines",[691],"2022-05-24","\n\nIn a GitLab book club, I recently read \"[The Laws of Simplicity](http://lawsofsimplicity.com/),\" a great book on a topic that has deeply fascinated me for many years. The book contains an acronym that expresses simplicity generation approaches: SHE, which stands for \"shrink, hide, embody.\" These three approaches for simplicity generation all share a common attribute: They are all creating illusions - not eliminations.\n\nI've seen this illusion repeat across many, many realms of pursuit for many years. Even in human language, vocabulary development, jargon, and acronyms all simply encapsulate worlds of complexity that still exist, but can be more easily referenced in a compact form that performs SHE on the world of concepts.\n\nAny illusion has a boundary or curtain where in front of the curtain the complexity can be dealt with by following simple rules, but, behind the curtain, the complexity must be managed by a stage manager. \n\nFor instance, when the magic show creates the spectre of sawing people in half, what appears to be a simple box is in fact an exceedingly elaborate contraption. Not only that, but the manufacturing process for an actual simple box and the sawing box are markedly different in terms of complexity. The manufacturing of complexity and its result are essentially the tradeoff for what would be the real-world complexity of actually sawing people in half and having them heal and stand up unharmed immediately afterward.\n\nTo bring this into the technical skills realm, consider that when you leverage a third-party component or API to add functionality, you only need to know the parameters to obtain the desired result. The people maintaining that component or API must know the quantum mechanics detail level of how to perform that work in a reliable and complete way.\n\nDocker containers are a mechanism for embodying complexity, and are used in scaled applications and within container-based CI. When a [CI/CD](/topics/ci-cd/) automation engineer uses container-based CI, it is possible to make things more complex and more expensive when attempting to do exactly the opposite.\n\nAt its core, this post is concerned with how it can happen that pursuing a simpler world through containers can turn into an antipattern - a reversal of desired outcomes - many times, without us noticing that the reversal is affecting our productivity. The prison of a paradigm is secure indeed.\n\n### The Second Law of Complexity Dynamics\n\nOver the years I have come to believe that the pursuit of reducing complexity has similar characteristics to [The Second Law of Thermodynamics](https://www.grc.nasa.gov/www/k-12/airplane/thermo2.html). The net result of a change between mass and energy results in the the same net amount of mass and energy, but their ratio and form have changed. In what I will coin \"The Second Law of Complexity Dynamics,\" complexity is similarly \"conserved,\" it is just reformed.\n\nIf complexity is not eliminated by simplifying efforts, we reduce its impact in a given realm by changing the ratio of complexity and simplicity on each side of one or more curtains. But alas, complexity did not die, it just hid and is now someone else's management challenge. It is important not to think of this as cheating. There is no question that hiding complexity carries the potential for massive efficiency gains when the world behind the hiding mechanisms becomes the realm of specialty skills and specialists. When it truly externalizes the complexity management for one party, the world becomes more simple for that party.\n\nHowever, the devil is in the details. If the hypothesis of \"no net elimination of complexity\" is correct, it is then important where the complexity migrates to. If it migrates to another part of the same process that must also be managed by the same people, then it may not result in a net gain of efficiency. If it migrates out of a previously embodied realm, then, in the pusuit of simplicity, we can actually reduce our overall efficiency when the process is considered as a whole.\n\n### Container-based CI pipelines as a useful case in point\n\nI see the potential for efficiency reversals to crop up in my daily work time and again, and an interesting place where I've seen it lately is in the tradeoff of linking together hyper-specialized modules of code in containers for CI versus leveraging more generalized modules.\n\nIn creating container-based pipelines, I experience the potential for an efficiency reversal I have to consciously manage.\n\nContainers make a simplicity tradeoff by design. They create a full runtime environment for a very single purpose but in doing so they strip back the container internals so far that general compute tasks are difficult inside them. If you step behind their \"complexity embodying\" curtain into the container, their simplistic environment can require more complex code to operate within.\n\nIn GitLab CI pipelines that utilize containers, all the scripts of jobs run inside the containers that are specified as their runtime environment. When one selects a specialized container - such as the alpine git container or the skopeo image management container - the code is subject to the limitations of the shell that container employs (if it has one at all).\n\nContainers were devised to be hyper-specialized, purpose-specific runtimes that assure they can always run and run quickly for scaled applications. However, for many containers this means no shell or a very stripped back shell like busybox sh. It frequently also means not including the package manager for the underlying Linux distribution.\n\nTime and again, I've found myself degrading the implementation of my shell code in key ways that make it more complex, so that it can run under these stripped back shells. In these cases, I do not benefit from the complexity hiding of newer versions of advanced shells like Bash v5. One of the areas is advanced Bash shell expansions, which embody a huge world of complex parsing and avoid a bunch of extraneous utilities. And another is advanced if and case statement comparison logic that processes regular expressions without external utilities and performs many other abstracted comparisons. There are many other areas of the language where this comes into play, but these two stand out.\n\n![](https://about.gitlab.com/images/blogimages/second-law-of-complexity-dynamics-container-pipeline-tradeoffs.png)\n\nSo by having a simpler shell like busybox sh, the simplicities of advanced shell features become *unhidden* and join my side of the curtain. Now I have to manage them in my code. But then, guess what? No package manager means the inability to install other Linux utilities and languages extensions that I could also employ to push that same complexity back out of my space. And, of course, it means installing Bash v5 would be difficult as well.\n\nSo the simplicity proposition of a tightly optimized purpose-specific container can reverse the purported efficiency gains in the very important realm of the code I have to write. It also means I frequently have to break up my code into multiple jobs to utilize the specializations of these containers in a sequence or to transport the results of a specialized container into a fuller coding environment. This increases the complexity of the pipeline as I now have to pass artifacts and variable data from one job to another with a host of additional YAML directives, and sometimes deploy infrastructure (e.g., [Runner caching](https://docs.gitlab.com/ee/ci/caching/#:~:text=For%20runners%20to%20work%20with,GitLab.com%20behave%20this%20way)).\n\nIn the case of CI using containers, when the simplicity tradeoffs move complexity to things I do not maintain, such as base containers, operating system packages, and full shell environments, into things I do maintain, such as CI YAML and Shell Script code, then I am also inheriting long-term complexity maintenance. In the cloud, we know this as undifferentiated heavy lifting.\n\nInterestingly, the proliferation of specialized containers can also require more machine resources and can lengthen processing time as containers are retrieved from registries and loaded and artifacts and source code are copied in and out of each job-based container.\n\n### Simplicity target: Efficiency\n\nIt's easy to lose sight of the amount of human effort and ingenuity being applied to knowing and managing the coding structure, rather than being applied to solving the real automation problems of the CI pipeline. The net complexity of the pipeline can also mean it is hard to maintain an understanding of it even if you are working in it every day - and for newcomers onboarding, it can be many weeks before they fully understand how the system works.\n\nOf course, I can create my own containers for CI pipelines, but now I've added the complexity of container development and continuous updates of the same in order for my pipeline code to be operational and stay healthy. I am still behind the curtain for that container. For teams whose software is not itself containerized, the prospect of learning to build containers just for CI can create a lot of understandable friction to adopting a container-based CI development process. This friction may be unnecessary if we make a key heuristic adaptation.\n\n### Walking the tightwire above the curtain\n\nSo how do I manage the tensions of these multiple worlds of complexity when it comes to container-based pipelines to try to avoid efficiency reversals in the net complexity of the pipeline?\n\nIt is simple. I will describe the method and then the key misapplied heuristic and how to adjust it.\n\n1. I hold that the primary benefits of container-based CI are a) dependency isolation by job (so that you don’t have a massive and brittle CI build machine specification to handle all possible build requirements), and b) clean CI build agent state by obtaining a clean container copy for each job. These benefits do not imply having to abide by microservices container resource planning and doing so is what creates an antipattern in my productivity.\n\n2. I frequently use a Bash 5 container (version pegged if need be) where all the complexity that advanced shell capabilities embody for me stay behind the curtain.\n\n3. Instead of running a hyper-minimalized container for a given utility, I do a runtime install of that utility (gasp!) in a container that has my rich shell. I utilize version pegging during the install if I feel version safety is paramount on the utility. Alternatively, if a very desirable runtime of some type is difficult to setup and does not have a package, I look for a container that has a package manager that matches a packaged version of the runtime and also allows me to install my advanced scripting language if needed.\n\n4. If, and only if, the net time of the needed runtime installs exceeds the net pipeline time to load a string of specialized containers (with artifact handling) plus my time to develop and manage a pipeline dependency in the form of a custom container, then do I consider possibly creating a pipeline specific container.\n\n5. Through this process a balancing principle also emerges. Since I have been doing runtime installs as a development practice, I have actually already MVPed what a pipeline specific container would need to have installed. I can literally copy the installation lines into a Docker file if I wish. I can also notice if I have commonality across multiple pipelines where it makes sense to create a multi-pipeline utility container.\n\nIn a recent project, following these principles caused me to avoid the skopeo container and instead install it on the Bash 5 container using a package manager.\n\nIf your team is big into Python or PowerShell as your CI language, it would make sense to start with recent releases of those containers. The point is not advanced Bash -but an advanced version of your general CI scripting language that prevents you from creating work arounds in your code for problems that are well-solved in publicly available runtimes.\n\nKeep in mind that this adjustment is very, very focused on containers **in CI pipelines**, which, by nature, reflect general compute processing requirements where many vastly different operations are required in a pipeline. I am not advocating this approach for true microservices applications where, by design, a given service has very defined purpose and characteristics and, at scale, massively benefits from the machine efficiency of hyper-minimalized, purpose-specific granularity.\n\n### Misapplied heuristics\n\nFrequently when a pattern has an inflection point at which it becomes an antipattern, it is due to misapplying the heuristics of the wrong realm. In this case, I believe, that normal containerization patterns for microservices apps are well founded, but they apply narrowly to \"engineered hyper-specialized compute\" of a granule we call \"a microservice\" (note the word \"micro\" applies to the scope of compute activities). Importantly, they apply because the process itself is designed as hyper-specialized around a very specific task. The container contents (included dependencies), immutability principle (no runtime change), and the runtime compute resources can be managed exceedingly minimally because of the small and highly specific scope of computing activities that occur within the process.\n\nThis is essentially the embodiment of the 12 Factor App principle called “[VIII. Concurrency](https://12factor.net/concurrency),” which asserts that scaling should be horizontal scaling of the same minimalized process, not vertical scaling of compute resources inside a given process. If the system experiences 10x work for a particular activity, we create 10 processes, we do not request 10x memory and 10x CPU within one running process. Microservices architecture tightly controls the amount of work in each request so that it is hyper-predictable in its compute resource requirements and, therefore, scalable by adding identical processes.\n\nCI compute, by nature, is the opposite of hyper-specialized. Across build, test, package, deploy, etc., etc., there are many huge variations in required machine resources of memory, CPU, network I/O and high-speed disk access and, importantly, included dependencies. The generalized compute nature also occurs due to varying inputs so the same defined process might need a lot more resources due to the nature of the raw input data. For example, varying input volume (e.g. a lot versus few data items) or varying input density (e.g. processing binary files versus text files). \n\nIt is the process that is being containerized that holds the attribute of generalized compute (bursty on at least some compute resources) or hyper-specialized (narrow definition of work to be done and therefore well-known compute resources per unit of completed work). Containerizing a process that exhibits generalized compute requirements is useful, but planning the resources of that container as if containerizing it has transformed the compute requirements into hyper-minimalized is the inflection point at which it becomes an antipattern, actually eroding the sought-after benefits we set out to create.\n\nIn the model I employ for leveraging containers in CI, the loosening of the hyper-specialization, immutablility (no-runtime installs), and very narrow compute resources principles of microservices simply reflects the real world in that CI compute as a whole exhibits the nature of generalized, not hyper-specialized, compute characteristics.\n\n> Another realm where this seems true is desired state configuration management technologies - also known as “Configuration as Code”. It is super simple if there are pre-existing components or recipes for all that you need to do but as soon as you have to build some for yourself, you enter a world of creating imperative code against a declarative API boundary (there's the \"embodiment\" curtain - the declarative API boundary). Generally, if you have not had to implement imperative code to process declaratively, this new world takes some significant experience to become proficient.\n\n### Iterating SA: Experimental improvements for your next project\n\n1. In general, favor simplicity boundaries that reduce your work, especially in the realm of undifferentiated heavy lifting. In the realm of container-based CI, this includes having a rich coding language and a package manager to acquire additional complexity embodying utilities quickly and easily.\n\n2. In general, be suspicious of an underlying antipattern if you have to spend an inordinate amount of time coding and maintaining workarounds in the service of simplicity. In the realm of container-based CI, this would be containers that are ultra-minimalized around microservices performance characteristics when they don’t hyper-scale as a standing service within CI.\n\n3. In general, stand back and examine the net complexity of the code and frameworks that will have to be maintained by yourself or your team and check if you’ve made tradeoffs that have a net negative tax on your efficiency. When complexity that can be managed by machines enters your workspace at high frequency, then you have a massive antipattern of human efficiency.\n\n4. It is frequent that when the hueristics being applied create negative human efficiency they also create negative machine efficiency. Watch for this effect in your projects. The diagram in the post shows that over-minimalized containers can easily lead to using a lot more of them - all of which has machine overhead as well.\n\nIf the above resonates, CI pipeline engineers might want to consider loosening the \"microservices\" heuristics of hyper-specialization, ultra-minimalization,  and immutability (no dynamic installs) for CI pipeline containers in order to ensure that the true net complexity level of the code they have to maintain is in balance and their productivity is preserved.\n\n### Appendix: Working examples of this idea\n\n- [AWS CLI Tools in Containers](https://gitlab.com/guided-explorations/aws/aws-cli-tools) has both Bash and PowerShell Core (on Linux OS) available so that one container set can suit the automation shell preference of both Linux and Windows heritage CI automation engineers.\n\n- CI file [installs yq dynamically](https://gitlab.com/guided-explorations/gl-k8s-agent/gitops/envs/world-greetings-env-1/-/blob/main/.gitlab-ci.yml#L47-48) in the Bash container, but then [only installs the heavier jq and skopeo](https://gitlab.com/guided-explorations/gl-k8s-agent/gitops/envs/world-greetings-env-1/-/blob/main/.gitlab-ci.yml#L63) if needed by the work implied, which demonstrates a way to be more efficient even when runtime installs are desired.\n\n- [Bash and PowerShell Script Code Libraries in Pure GitLab CI YAML](https://gitlab.com/guided-explorations/ci-cd-plugin-extensions/script-code-libraries-in-pure-gitlab-ci-yaml) shows how to have libraries of CI script code available to every container in a pipeline without encapsulating the libraries in a container themselves and with minimalized CI YAML complexity compared to YAML anchors, references, or extends. While the method is a little bit challenging to setup, from then on out it pays back by decoupling scripting libraries from any other pipeline artifact.\n\n- [CI/CD Extension Freemarker File Templating](https://gitlab.com/guided-explorations/ci-cd-plugin-extensions/ci-cd-plugin-extension-freemarker-file-templating) shows the install is very quick and only affects one job and still version pegs the installed utility.\n",[787,788,9,809,1016],{"slug":3821,"featured":6,"template":700},"second-law-of-complexity-dynamics","content:en-us:blog:second-law-of-complexity-dynamics.yml","Second Law Of Complexity Dynamics","en-us/blog/second-law-of-complexity-dynamics.yml","en-us/blog/second-law-of-complexity-dynamics",{"_path":3827,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3828,"content":3834,"config":3839,"_id":3841,"_type":13,"title":3842,"_source":15,"_file":3843,"_stem":3844,"_extension":18},"/en-us/blog/secure-and-publish-python-packages-a-guide-to-ci-integration",{"title":3829,"description":3830,"ogTitle":3829,"ogDescription":3830,"noIndex":6,"ogImage":3831,"ogUrl":3832,"ogSiteName":685,"ogType":686,"canonicalUrls":3832,"schema":3833},"Secure and publish Python packages: A guide to CI integration","Learn how to implement a secure CI/CD pipeline across five stages with the GitLab DevSecOps platform.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662080/Blog/Hero%20Images/AdobeStock_1097303277.jpg","https://about.gitlab.com/blog/secure-and-publish-python-packages-a-guide-to-ci-integration","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Secure and publish Python packages: A guide to CI integration\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tim Rizzi\"}],\n        \"datePublished\": \"2025-01-21\",\n      }",{"title":3829,"description":3830,"authors":3835,"heroImage":3831,"date":3836,"body":3837,"category":830,"tags":3838},[932],"2025-01-21","Supply chain security is a critical concern in software development.\nOrganizations need to verify the authenticity and integrity of their\nsoftware packages. This guide will show you how to implement a secure CI/CD\npipeline for Python packages using GitLab CI, incorporating package signing\nand attestation using Sigstore's Cosign.\n\n\nYou'll learn:\n\n\n- [Why sign and attest your Python\npackages?](#why-sign-and-attest-your-python-packages%3F)\n\n- [Pipeline overview](#pipeline-overview)\n\n- [Complete pipeline implementation: Setting up the\nenvironment](#complete-pipeline-implementation-setting-up-the-environment)\n   * [Environment configuration](#environment-configuration)\n   * [Configuration breakdown](#configuration-breakdown)\n-  The 6 stages\n\n    1. [Building](#building-crafting-the-package)\n    2. [Signing](#signing-the-digital-notarization)\n    3. [Verification](#verification-the-security-checkpoint)\n    4. [Publishing](#publishing-the-controlled-release)\n    5. [Publishing signatures](#publishing-signatures-making-verification-possible)\n    6. [Consumer verification](#consumer-verification-testing-the-user-experience)\n\n## Why sign and attest your Python packages?\n\n\nHere are four reasons to sign and attest your Python packages:\n\n\n* **Supply chain security:** Package signing ensures that the code hasn't\nbeen tampered with between build and deployment, protecting against supply\nchain attacks.\n\n* **Compliance requirements:** Many organizations, especially in regulated\nindustries, require cryptographic signatures and provenance information for\nall deployed software.\n\n* **Traceability:** Attestations provide a verifiable record of build\nconditions, including who built the package and under what circumstances.\n\n* **Trust verification:** Consumers of your package can cryptographically\nverify its authenticity before installation.\n\n\n## Pipeline overview\n\n\nEnsuring your code's integrity and authenticity is necessary. Imagine a\npipeline that doesn't just compile your code but creates a cryptographically\nverifiable narrative of how, when, and by whom your package was created.\nEach stage acts as a guardian, checking and documenting the package's\nprovenance.\n\n\nHere are six stages of a GitLab pipeline that ensure your package is secure\nand trustworthy:\n\n\n* Build: Creates a clean, standard package that can be easily shared and\ninstalled.\n\n* Signing: Adds a digital signature that proves the package hasn't been\ntampered with since it was created.\n\n* Verification: Double-checks that the signature is valid and the package\nmeets all our security requirements.\n\n* Publishing: Uploads the verified package to GitLab's package registry,\nmaking it available for others to use.\n\n* Publishing Signatures: Makes signatures available for verification.\n\n* Consumer Verification: Simulates how end users can verify package\nauthenticity.\n\n\n## Complete pipeline implementation: Setting up the environment\n\n\nBefore we build our package, we need to set up a consistent and secure build\nenvironment. This configuration ensures every package is created with the\nsame tools, settings, and security checks.\n\n\n### Environment configuration\n\n\nOur pipeline requires specific tools and settings to work correctly.\n\n\nPrimary configurations:\n\n\n* Python 3.10 for consistent builds\n\n* Cosign 2.2.3 for package signing\n\n* GitLab package registry integration\n\n* Hardcoded package version for reproducibility\n\n\n**Note about versioning:** We've chosen to use a hardcoded version\n(`\"1.0.0\"`) in this example rather than deriving it from git tags or\ncommits. This approach ensures complete reproducibility and makes the\npipeline behavior more predictable. In a production environment, you might\nwant to use semantic versioning based on git tags or another versioning\nstrategy that fits your release process.\n\n\nTool requirements:\n\n\n* Basic utilities: `curl`, `wget`\n\n* Cosign for cryptographic signing\n\n* Python packaging tools: `build`, `twine`, `setuptools`, `wheel`\n\n\n### Configuration breakdown\n\n\n```yaml\n\nvariables:\n  PYTHON_VERSION: '3.10'\n  PACKAGE_NAME: ${CI_PROJECT_NAME}\n  PACKAGE_VERSION: \"1.0.0\"\n  FULCIO_URL: 'https://fulcio.sigstore.dev'\n  REKOR_URL: 'https://rekor.sigstore.dev'\n  CERTIFICATE_IDENTITY: 'https://gitlab.com/${CI_PROJECT_PATH}//.gitlab-ci.yml@refs/heads/${CI_DEFAULT_BRANCH}'\n  CERTIFICATE_OIDC_ISSUER: 'https://gitlab.com'\n  PIP_CACHE_DIR: \"$CI_PROJECT_DIR/.pip-cache\"\n  COSIGN_YES: \"true\"\n  GENERIC_PACKAGE_BASE_URL: \"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${PACKAGE_VERSION}\"\n```\n\n\nWe use caching to speed up subsequent builds:\n\n\n```yaml\n\ncache:\n  paths:\n    - ${PIP_CACHE_DIR}\n```\n\n\n## Building: Crafting the package\n\n\nEvery software journey begins with creation. In our pipeline, the build\nstage is where raw code transforms into a distributable package, ready to\ntravel across different Python environments.\n\n\nThe build process creates two standardized formats:\n\n\n* a wheel package (.whl) for quick, efficient installation\n\n* a source distribution (.tar.gz) that carries the complete code\n\n\nHere's the build stage implementation:\n\n\n```yaml\n\nbuild:\n  extends: .python-job\n  stage: build\n  script:\n    - git init\n    - git config --global init.defaultBranch main\n    - git config --global user.email \"ci@example.com\"\n    - git config --global user.name \"CI\"\n    - git add .\n    - git commit -m \"Initial commit\"\n    - export NORMALIZED_NAME=$(echo \"${CI_PROJECT_NAME}\" | tr '-' '_')\n    - sed -i \"s/name = \\\".*\\\"/name = \\\"${NORMALIZED_NAME}\\\"/\" pyproject.toml\n    - sed -i \"s|\\\"Homepage\\\" = \\\".*\\\"|\\\"Homepage\\\" = \\\"https://gitlab.com/${CI_PROJECT_PATH}\\\"|\" pyproject.toml\n    - python -m build\n  artifacts:\n    paths:\n      - dist/\n      - pyproject.toml\n```\n\n\nLet's break down what this build stage does:\n\n\n1. Initializes a Git repository (`git init`) and configures it with basic\nsettings\n\n2. Normalizes the package name by converting hyphens to underscores, which\nis required for Python packaging\n\n3. Updates the package metadata in `pyproject.toml` to match our project\nsettings\n\n4. Builds both wheel and source distribution packages using `python -m\nbuild`\n\n5. Preserves the built packages and configuration as artifacts for\nsubsequent stages\n\n\n## Signing: The digital notarization\n\n\nIf attestation is the package's biography, signing is its cryptographic seal\nof authenticity. This is where we transform our package from a mere\ncollection of files into a verified, tamper-evident artifact.\n\n\nThe signing stage uses Cosign to apply a digital signature as an unbreakable\nseal. This isn't just a stamp — it's a complex cryptographic handshake that\nproves the package's integrity and origin.\n\n\n```yaml\n\nsign:\n  extends: .python+cosign-job\n  stage: sign\n  id_tokens:\n    SIGSTORE_ID_TOKEN:\n      aud: sigstore\n  script:\n    - |\n      for file in dist/*.whl dist/*.tar.gz; do\n        if [ -f \"$file\" ]; then\n          filename=$(basename \"$file\")\n          cosign sign-blob --yes \\\n            --fulcio-url=${FULCIO_URL} \\\n            --rekor-url=${REKOR_URL} \\\n            --oidc-issuer $CI_SERVER_URL \\\n            --identity-token $SIGSTORE_ID_TOKEN \\\n            --output-signature \"dist/${filename}.sig\" \\\n            --output-certificate \"dist/${filename}.crt\" \\\n            \"$file\"\n        fi\n      done\n  artifacts:\n    paths:\n      - dist/\n```\n\n\nThis signing stage performs several crucial operations:\n\n\n1. Obtains an OIDC token from GitLab for authentication with Sigstore\nservices\n\n2. Processes each built package (both wheel and source distribution)\n\n3. Uses Cosign to create a cryptographic signature (`.sig`) for each package\n\n4. Generates a certificate (`.crt`) that proves the signature's authenticity\n\n5. Stores both signatures and certificates alongside the packages as\nartifacts\n\n\n## Verification: The security checkpoint\n\n\nVerification is our final quality control gate. It's not just a check — it's\na security interrogation where every aspect of the package is scrutinized.\n\n\n```yaml\n\nverify:\n  extends: .python+cosign-job\n  stage: verify\n  script:\n    - |\n      failed=0\n      for file in dist/*.whl dist/*.tar.gz; do\n        if [ -f \"$file\" ]; then\n          filename=$(basename \"$file\")\n          if ! cosign verify-blob \\\n            --signature \"dist/${filename}.sig\" \\\n            --certificate \"dist/${filename}.crt\" \\\n            --certificate-identity \"${CERTIFICATE_IDENTITY}\" \\\n            --certificate-oidc-issuer \"${CERTIFICATE_OIDC_ISSUER}\" \\\n            \"$file\"; then\n            failed=1\n          fi\n        fi\n      done\n      if [ $failed -eq 1 ]; then\n        exit 1\n      fi\n```\n\n\nThe verification stage implements several security checks:\n\n\n1. Examines each package file in the `dist` directory\n\n2. Uses Cosign to verify the signature matches the package content\n\n3. Confirms the certificate's identity matches our expected GitLab pipeline\nidentity\n\n4. Validates our trusted OIDC provider issued the certificate\n\n5. Fails the entire pipeline if any verification check fails, ensuring only\nverified packages proceed\n\n\n## Publishing: The controlled release\n\n\nPublishing is where we make our verified packages available through GitLab's\npackage registry. It's a carefully choreographed release that ensures only\nverified, authenticated packages reach their destination.\n\n\n```yaml\n\npublish:\n  extends: .python-job\n  stage: publish\n  script:\n    - |\n      cat \u003C\u003C EOF > ~/.pypirc\n      [distutils]\n      index-servers = gitlab\n      [gitlab]\n      repository = ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi\n      username = gitlab-ci-token\n      password = ${CI_JOB_TOKEN}\n      EOF\n      TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token \\\n        twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi \\\n        dist/*.whl dist/*.tar.gz\n```\n\n\nThe publishing stage handles several important tasks:\n\n\n1. Creates a `.pypirc` configuration file with GitLab package registry\ncredentials\n\n2. Uses the GitLab CI job token for secure authentication\n\n3. Uploads both wheel and source distribution packages to the GitLab PyPI\nregistry\n\n4. Makes the packages available for installation via pip\n\n\n## Publishing signatures: Making verification possible\n\n\nAfter publishing the packages, we must make their signatures and\ncertificates available for verification. We store these in GitLab's generic\npackage registry, making them easily accessible to users who want to verify\npackage authenticity.\n\n\n```yaml\n\npublish_signatures:\n  extends: .python+cosign-job\n  stage: publish_signatures\n  script:\n    - |\n      for file in dist/*.whl dist/*.tar.gz; do\n        if [ -f \"$file\" ]; then\n          filename=$(basename \"$file\")\n          curl --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \\\n               --fail \\\n               --upload-file \"dist/${filename}.sig\" \\\n               \"${GENERIC_PACKAGE_BASE_URL}/${filename}.sig\"\n\n          curl --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \\\n               --fail \\\n               --upload-file \"dist/${filename}.crt\" \\\n               \"${GENERIC_PACKAGE_BASE_URL}/${filename}.crt\"\n        fi\n      done\n```\n\n\nThe signature publishing stage performs these key operations:\n\n\n1. Processes each built package to find its corresponding signature files\n\n2. Uses the GitLab API to upload the signature (`.sig`) file to the generic\npackage registry\n\n3. Uploads the corresponding certificate (`.crt`) file\n\n4. Makes these verification artifacts available for downstream package\nconsumers\n\n5. Uses the same version and package name to maintain the connection between\npackages and signatures\n\n\n## Consumer verification: Testing the user experience\n\n\nThe final stage simulates how end users will verify your package's\nauthenticity. This stage acts as a final check and a practical example of\nthe verification process.\n\n\n```yaml\n\nconsumer_verification:\n  extends: .python+cosign-job\n  stage: consumer_verification\n  script:\n    - |\n      git init\n      git config --global init.defaultBranch main\n      mkdir -p pkg signatures\n\n      pip download --index-url \"https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi/simple\" \\\n          \"${NORMALIZED_NAME}==${PACKAGE_VERSION}\" --no-deps -d ./pkg\n\n      pip download --no-binary :all: \\\n          --index-url \"https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi/simple\" \\\n          \"${NORMALIZED_NAME}==${PACKAGE_VERSION}\" --no-deps -d ./pkg\n\n      failed=0\n      for file in pkg/*.whl pkg/*.tar.gz; do\n        if [ -f \"$file\" ]; then\n          filename=$(basename \"$file\")\n          sig_url=\"${GENERIC_PACKAGE_BASE_URL}/${filename}.sig\"\n          cert_url=\"${GENERIC_PACKAGE_BASE_URL}/${filename}.crt\"\n\n          curl --fail --silent --show-error \\\n               --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \\\n               --output \"signatures/${filename}.sig\" \\\n               \"$sig_url\"\n\n          curl --fail --silent --show-error \\\n               --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \\\n               --output \"signatures/${filename}.crt\" \\\n               \"$cert_url\"\n\n          if ! cosign verify-blob \\\n            --signature \"signatures/${filename}.sig\" \\\n            --certificate \"signatures/${filename}.crt\" \\\n            --certificate-identity \"${CERTIFICATE_IDENTITY}\" \\\n            --certificate-oidc-issuer \"${CERTIFICATE_OIDC_ISSUER}\" \\\n            \"$file\"; then\n            failed=1\n          fi\n        fi\n      done\n\n      if [ $failed -eq 1 ]; then\n        exit 1\n      fi\n```\n\n\nThis consumer verification stage simulates the end-user experience by:\n\n\n1. Creating a clean environment to test package installation\n\n2. Downloading the published packages from the GitLab PyPI registry\n\n3. Retrieving the corresponding signatures and certificates from the generic\npackage registry\n\n4. Performing the same verification steps that end users would perform\n\n5. Ensuring the entire process works from a consumer's perspective\n\n6. Failing the pipeline if any verification step fails, providing an early\nwarning of any issues\n\n\n## Summary\n\n\nThis comprehensive pipeline provides a secure and reliable way to build,\nsign, and publish Python packages to GitLab's package registry. By following\nthese practices and implementing the suggested security measures, you can\nensure your packages are appropriately verified and safely distributed to\nyour users.\n\n\nThe pipeline combines modern security practices with efficient automation to\ncreate a robust software supply chain. Using Sigstore's Cosign for signing\nand attestation, along with GitLab's built-in security features, you can\nprovide users with trustworthy cryptographically verified packages.\n\n\n> #### Get started on your security journey today with a [free trial\nof GitLab\nUltimate](https://gitlab.com/-/trials/new?glm_content=default-saas-trial&glm_source=about.gitlab.com).\n\n\n## Learn more\n\n- [Documentation: Use Sigstore for keyless signing and\nverification](https://docs.gitlab.com/ee/ci/yaml/signing_examples.html)\n\n- [Streamline security with keyless signing and verification in\nGitLab](https://about.gitlab.com/blog/keyless-signing-with-cosign/)\n\n- [Annotate container images with build provenance using Cosign in GitLab\nCI/CD](https://about.gitlab.com/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd/)\n",[830,232,283,696,787,108,495,9,1016],{"slug":3840,"featured":90,"template":700},"secure-and-publish-python-packages-a-guide-to-ci-integration","content:en-us:blog:secure-and-publish-python-packages-a-guide-to-ci-integration.yml","Secure And Publish Python Packages A Guide To Ci Integration","en-us/blog/secure-and-publish-python-packages-a-guide-to-ci-integration.yml","en-us/blog/secure-and-publish-python-packages-a-guide-to-ci-integration",{"_path":3846,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3847,"content":3853,"config":3858,"_id":3860,"_type":13,"title":3861,"_source":15,"_file":3862,"_stem":3863,"_extension":18},"/en-us/blog/secure-and-safe-login-and-commits-with-gitlab-yubico",{"title":3848,"description":3849,"ogTitle":3848,"ogDescription":3849,"noIndex":6,"ogImage":3850,"ogUrl":3851,"ogSiteName":685,"ogType":686,"canonicalUrls":3851,"schema":3852},"Secure and safe login and commits with GitLab + Yubico","Learn how GitLab and Yubico have partnered to strengthen software development security through robust authentication measures.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663259/Blog/Hero%20Images/REFERENCE_-_display_preview_for_blog_images__3_.png","https://about.gitlab.com/blog/secure-and-safe-login-and-commits-with-gitlab-yubico","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Secure and safe login and commits with GitLab + Yubico\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-04-02\",\n      }",{"title":3848,"description":3849,"authors":3854,"heroImage":3850,"date":3855,"body":3856,"category":830,"tags":3857},[1444],"2025-04-02","We live in a time where data breaches and phishing attacks make daily\nheadlines. These breaches can cause harm to an organization, such as\nregulatory fines, business downtime, or even worse, reputational damage. In\nterms of authentication, passwords have been the backbone of online security\nfor decades, however, they're increasingly proving inadequate against\nsophisticated cyber threats.\n\n\nGitLab and [Yubico](https://www.yubico.com/) have partnered to strengthen\nsoftware development security through robust authentication measures. Yubico\nis the inventor of the YubiKey, a hardware security key that delivers\nphishing-resistant multi-factor authentication (MFA). By implementing FIDO\nUniversal 2nd Factor (U2F) and YubiKey hardware protection, GitLab offers\ndevelopers a powerful defense against phishing attacks and other cyber\nthreats, ensuring their code and projects remain secure. This collaboration\nexpands enterprise-grade authentication in the GitLab platform, allowing\nprogrammers to focus on creating software while maintaining confidence in\ntheir account's integrity.\n\n\nThis article explains how to configure GitLab to use YubiKeys to protect\ndevelopers from online threats. You’ll also learn how to further prevent\ntampering with GitLab verified commits.\n\n\n## How YubiKeys work\n\n\nAt their core, YubiKeys function as cryptographic hardware tokens that\ngenerate and store private keys in a secure element. These keys implement\nFIDO2/WebAuthn authentication protocols, which can be used as an additional\nfactor to login to GitLab.\n\n\nHere's how it works when logging in:\n\n\n1. You enter your username and password.  \n\n2. GitLab sends a cryptographic challenge to your browser.  \n\n3. Your browser requests the YubiKey to sign this challenge.  \n\n4. You physically touch the YubiKey to approve.\n\n5. The YubiKey creates a unique cryptographic signature for that specific\nservice and challenge.  \n\n6. GitLab verifies the signature using your public key stored during setup.\n\n\nMost major security breaches involve compromised passwords. Adding a YubiKey\nsecures your account from a remote breach, even if your password is stolen,\nso you can rest assured that your GitLab account is secure. Additional key\nsecurity benefits of using YubiKey for authentication with GitLab include:\n\n\n* **Phishing protection:** Fake sites won't have the correct cryptographic\nkeys to verify the response. \n\n* **No secrets to steal:** The private key never leaves the YubiKey.  \n\n* **Physical security:** Physical presence is required to use it (you must\ntouch the YubiKey).\n\n\n## Setting up YubiKey multifactor authentication in GitLab\n\n\nNow let’s go over how to set up a Yubikey for multifactor authentication in\nGitLab. Make sure you're using a [supported browser and operating\nsystem](https://support.yubico.com/hc/en-us/articles/360016615020-Operating-system-and-web-browser-support-for-FIDO2-and-U2F)\nas they have better WebAuthn support for hardware security keys.\n\n\n1. First, log in to your GitLab account and go to your user settings (click\nyour avatar in the top left corner and select **Preferences**). \n\n2. In the left sidebar, click on **Account** and navigate to the\n**Two-factor Authentication** section.\n\n3. If you haven't already enabled 2FA, you'll need to do that first.\n\n    a. Click **Enable two-factor authentication**.\n\n    b. Scan the QR code with your authenticator app.\n\n    c. Enter the code from your authenticator app.\n\n    d. Enter your GitLab password. If you ever need to access your GitLab account without using Google authentication, you may need to:\n    * Use the **Forgot password** option on the GitLab login page to set up a separate GitLab password.\n    * Contact your GitLab administrator to help you set up alternative login methods.\n\n   e. Save your recovery codes in a safe place.\n\n4. Once 2FA is enabled, go back to the previous screen by pressing **Manage\ntwo-factor authentication** and scroll down to the **Register hardware\ntoken** section.  \n\n5. Press the **Set up new device** button.  \n    a. A popup from your browser should appear. **Note:** This image may look different depending on your browser. You may also get popups from password managers feel free to ignore them. \n\n![Browser (Brave) Auth\nRequest](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674606/Blog/Content%20Images/browser_auth_request.png)\n\n\n    b. Select **Use a phone, tablet, or security key**.\n\n\n6. A new popup will appear.\n\n\n![browser security key\nrequest](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674607/Blog/Content%20Images/browser_security_key_request.png)\n\n\n    a. Insert your YubiKey into your computer's USB port.\n\n\n    b. Touch the metal contact/button on your YubiKey when\nprompted. The field will automatically fill with a one-time code.\n\n\n7. Enter your GitLab Password and provide a name for your Hardware Key.  \n\n8. Click **Register** to add the YubiKey to your account.\n\n\nCongratulations, your YubiKey is now registered and can be used as a second\nfactor when logging into GitLab! You can register multiple YubiKeys to your\naccount for backup purposes. **Note:** The process may vary slightly among\nbrowsers.\n\n\n![yubikey\nregistered](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674607/Blog/Content%20Images/yubikey_registered.png)\n\n\n\u003Ccenter>\u003Ci>YubiKey registered successfully\u003C/i>\u003C/center>\n\n\n## Signing in with a YubiKey\n\n\nNow that we have our YubiKey configured, we can log in as follows:\n\n\n1. Go to GitLab.com.\n\n\n![GitLab\nlogin](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674607/Blog/Content%20Images/gitlab_login.png)\n\n\n2. Provide your username and password and then press the **Sign in** button.\n\n3. You will be sent to the following screen.\n\n\n![GitLab 2fa\nlogin](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674606/Blog/Content%20Images/2fa_login.png)\n\n\n    a. A popup, like the one below, should come up. **Note:** This\nimage may look different depending on your browser. You may also get popups\nfrom password managers; feel free to ignore them.\n\n\n![Browser security key\nrequest](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674607/Blog/Content%20Images/browser_security_key_request.png)\n\n\n    b. Insert your YubiKey into your computer's USB port.\n\n\n    c. Touch the metal contact/button on your YubiKey when\nprompted. The field will automatically fill with a one-time code.\n\n\nNow, you should be logged in and taken to your GitLab page. **Note:** The\nprocess may vary slightly among browsers.\n\n\n## What happens if I lose my YubiKey?\n\n\nYubico recommends that you use and keep a backup YubiKey. When considering\nyour home, car, or office, you wouldn’t think twice about having a backup\nkey to keep in a safe place. Your digital self should get the same level of\nconsideration. A backup YubiKey kept in a safe place provides a quick and\nsafe backup if your primary YubiKey is lost. Keeping a backup will also\neasily enable you to deactivate the lost YubiKey and add a new primary or\nsecondary YubiKey.\n\n\nIf you do not have an additional YubiKey added, it is recommended to have\nanother form of 2FA added to your accounts. In either case, you should be\nable to get access to your account and remove the lost key from the account.\nPlease note that if a spare key or another authentication method hasn’t been\nadded, you will need to contact the service/website for help with recovering\nyour account.\n\n\n## GitLab verified commits\n\n\nTo further prevent tampering, you can also configure verified commits.\nVerified commits in GitLab use GPG (GNU Privacy Guard) signatures to prove\nthat a commit actually came from you. This adds another layer of security on\ntop of authentication by ensuring that not only is your account secure, but\nevery code change can be cryptographically verified as coming from you.\n\n\nYour YubiKey can store GPG keys:\n\n\n* The private key is stored securely on the YubiKey.  \n\n* The public key is shared with GitLab.\n\n* The key pair is used to sign your commits.\n\n\nOnce the GPG keys have been set up:\n\n\n* When you make a commit, Git uses your private key to create a signature.  \n\n* The GPG key is accessed from the attached YubiKey.  \n\n* The signature is stored with the commit metadata.  \n\n* GitLab verifies the signature using your public key.\n\n\n## Setting up verified commits\n\n\nLet’s go over how to configure verified commits. In this example, the GPG\nkey will live inside your YubiKey, providing an extra layer of security.\n\n\n1. Install required software.\n\n\n```bash\n\n# On macOS\n\nbrew install --cask yubico-yubikey-manager\n\nbrew install gnupg gpg yubikey-manager\n\n\n# On Ubuntu/Debian\n\nsudo apt install gnupg gpg yubikey-personalization\n\n\n# On Windows\n\n# Download and install Gpg4win from https://gpg4win.org\n\n```\n\n\n2. Check YubiKey GPG status.\n\n\n```bash\n\ngpg --card-status\n\n```\n\n3. Generate GPG keys directly on YubiKey (more secure).\n\n\n```bash\n\n# Start GPG edit mode\n\ngpg --card-edit\n\n\n# Enter admin mode\n\nadmin\n\n\n# Generate key directly on card\n\n# PIN = '123456' | Admin PIN = '12345678'\n\ngenerate\n\n\n# Follow prompts\n\n# See documentation for more info \n\n#\nhttps://support.yubico.com/hc/en-us/articles/360013790259-Using-Your-YubiKey-with-OpenPGP\n\n```\n\n\n4. Export your public key.\n\n\n```bash\n\n# Get your key ID\n\ngpg --list-secret-keys --keyid-format LONG\n\n\n# Export the public key\n\ngpg --armor --export YOUR_KEY_ID\n\n```\n\n\n5. Add the public key to GitLab.\n\n    a. Click on your GitLab Avatar and select **Preferences**.\n\n    b. On the side tab select **GPG Keys**.\n\n    c. Click **Add new key**.\n\n    d. Paste your public key.\n\n    e. Click **Add key**.\n\n6. Configure Git.\n\n\n```bash\n\n# Set signing key\n\ngit config --global user.signingkey YOUR_KEY_ID\n\n\n# Enable automatic signing\n\ngit config --global commit.gpgsign true\n\n\n# Tell GPG which key to use\n\necho \"default-key YOUR_KEY_ID\" >> ~/.gnupg/gpg.conf\n\n```\n\n\n7. Now let’s test the configuration by creating a test commit in a project:\n\n\n```bash\n\n# Make a change in the project\n\n# Add changes\n\ngit add .\n\n\n# Make a test commit\n\ngit commit -S -m \"Test signed commit\"\n\n\n# Verify signature\n\ngit verify-commit HEAD\n\n\n# Push the change\n\ngit push\n\n```\n\n\nThe `git verify-commit HEAD` command should show the GPG key used:\n\n\n```bash\n\ngpg: Signature made Wed Feb 26 11:45:00 2025 CST\n\ngpg:                using RSA key YOUR_KEY_ID\n\ngpg: Good signature from “NAME (DESCRIPTION) \u003CEMAIL>\" [ultimate]\n\n```\n\n\nThen, when viewing the commit in GitLab, you should now see that the commit\nis verified as follows:\n\n\n![Commit is\nverified](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674607/Blog/Content%20Images/verified.png)\n\n\n\u003Ccenter>\u003Ci>Commit verified with GPG key\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nYou can also use the [commits\nAPI](https://docs.gitlab.com/api/commits/#get-signature-of-a-commit) to\ncheck a commit’s signature allowing you to further operationalize the\nverification workflow.\n\n\n## Learn more\n\n\nTo learn more about GitLab, Yubico, and the solutions each provides, check\nout these resources:\n\n\n* [Why GitLab](https://about.gitlab.com/why-gitlab/)  \n\n* [Why Yubico](https://www.yubico.com/why-yubico/)  \n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab listing in the \"Works with YubiKey\"\ncatalog](https://www.yubico.com/works-with-yubikey/catalog/gitlab/)  \n\n* [Verified Commits - GitLab\ndocumentation](https://docs.gitlab.com/ee/user/project/repository/signed_commits/)  \n\n* [Push Rules in\nGitLab](https://docs.gitlab.com/user/project/repository/push_rules/)  \n\n* [Sign Commit with GPG Keys\ndocumentation](https://docs.gitlab.com/user/project/repository/signed_commits/gpg/)\n",[232,830,9,495,694,696],{"slug":3859,"featured":90,"template":700},"secure-and-safe-login-and-commits-with-gitlab-yubico","content:en-us:blog:secure-and-safe-login-and-commits-with-gitlab-yubico.yml","Secure And Safe Login And Commits With Gitlab Yubico","en-us/blog/secure-and-safe-login-and-commits-with-gitlab-yubico.yml","en-us/blog/secure-and-safe-login-and-commits-with-gitlab-yubico",{"_path":3865,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3866,"config":3870,"content":3872,"_id":3878,"_type":13,"title":3879,"_source":15,"_file":3880,"_stem":3881,"_extension":18},"/en-us/blog/secure-rust-development-with-gitlab",{"config":3867,"title":3868,"description":3869},{"noIndex":6},"Secure Rust development with GitLab","Learn how GitLab supports Rust development through its CI/CD capabilities, security scanning, dedicated Rust integrations, AI features, and more.",{"slug":3871,"featured":90,"template":700},"secure-rust-development-with-gitlab",{"title":3868,"description":3869,"authors":3873,"tags":3874,"heroImage":3875,"category":785,"date":3876,"body":3877},[1444],[268,743,9],"https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314674/tct6zf6evw0xgddd2vo3.png","2025-09-02","Rust has emerged as one of the most beloved programming languages due to its performance,\nmemory safety, and concurrency features. As Rust adoption continues to grow, many developers\nare looking for robust CI/CD platforms to support their Rust projects.\n\nGitLab's appeal to Rust developers extends beyond simple code hosting. The platform offers\nrobust [CI/CD](https://about.gitlab.com/topics/ci-cd/) capabilities that align perfectly with Rust's emphasis on safety, performance, and reliability. GitLab makes it easy to create repositories and use off-the-shelf Docker\ncontainers to put together custom CI jobs. Developers can easily set up automated testing,\ncross-platform builds, and documentation generation. The platform's integrated approach to\nDevSecOps resonates with Rust's philosophy of providing comprehensive tooling out of the box.\n\n## About the demo application\n\nBeing interested in how mortgage rates affect monthly payments and how hard it is to\nto afford a house in the current times, I decided to write a [mortgage calculator](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/rust/mortgage-calculator) in Rust,\nwhich I will use as an example throughout this tutorial. Feel free to [import this project](https://docs.gitlab.com/user/project/import/repo_by_url/) and follow along.\n\nThe mortgage calculator will help users calculate monthly mortgage payments,\nincluding principal, interest, property taxes, insurance, PMI, and HOA fees. It provides a\nmodern, intuitive GUI using the [egui](https://www.egui.rs/) framework, as well as a CLI for running it in the terminal.\n\n![Mortgage calculator GUI](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314538/l5bjnzqvpoyikuyxpx2a.png)\n\nThis application contains a [`.gitlab-ci.yml`](https://gitlab.com/gitlab-da/tutorials/security-and-governance/devsecops/rust/mortgage-calculator/-/blob/main/.gitlab-ci.yml?ref_type=heads) that generates a pipeline, which will build, test,\npackage, scan, and deploy the software. We will go over this pipeline definition in detail in the\nsections below.\n\n![Mortgage Calculator Pipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314540/llmsfoaupedhkem0hjqp.png)\n\n## Building and testing Rust applications\n\nGitLab's Docker-based CI/CD system excels at Rust development workflows, providing a robust\nfoundation for compilation, testing, and code quality checks. The platform's caching mechanisms\nare particularly valuable for Rust projects, which can have lengthy compilation times due to the\nlanguage's thorough optimization and safety checking processes.\n\n### Building\n\nRust's excellent cross-compilation capabilities combined with GitLab's flexible CI/CD system\ncreate a powerful solution for building applications across multiple platforms. This is particularly\nvaluable for Rust applications that need to run on various operating systems and architectures without\nsacrificing performance or requiring platform-specific code.\n\n**Note:** You can learn more about the `.gitlab-ci.yml` by reading the [CI/CD YAML syntax reference](https://docs.gitlab.com/ci/yaml/).\n\n```yaml\n# Cache configuration to speed up builds by reusing dependencies\ncache:\n  key: $CI_COMMIT_REF_SLUG                               # Use branch name as cache key\n  paths:\n    - .cargo/                                            # Cache Cargo registry and git dependencies\n    - target/                                            # Cache compiled artifacts\n\n# Base template for Rust jobs - shared configuration\n.rust-template:\n  image: rust:$RUST_VERSION-slim                         # Use slim Rust image for faster downloads\n  before_script:\n    # Install system dependencies required for building the Rust application\n    - apt-get update && apt-get install -y pkg-config libssl-dev libgtk-3-dev libxcb-shape0-dev libxcb-xfixes0-dev\n\n# Template for cross-compilation build jobs\n.build-template:\n  extends: .rust-template                                # Inherit from rust-template\n  stage: build                                           # Execute during build stage\n  script:\n    - rustup target add $TARGET                          # Add the target platform for cross-compilation\n    - cargo build --release --target $TARGET             # Build optimized release binary for target platform\n\n# Build for Linux x86_64 (primary target platform)\nbuild-linux:\n  extends: .build-template                               # Use build template configuration\n  variables:\n    TARGET: x86_64-unknown-linux-gnu                     # Linux 64-bit target\n  artifacts:\n    paths:\n      - target/$TARGET/release/mortgage-calculator       # Save the compiled binary\n    expire_in: 1 week                                    # Keep artifacts for 1 week\n  allow_failure: false                                   # This build must succeed\n\n# Build for Windows x86_64 (cross-compilation)\nbuild-windows:\n  extends: .build-template                               # Use build template configuration\n  variables:\n    TARGET: x86_64-pc-windows-gnu                        # Windows 64-bit target\n  artifacts:\n    paths:\n      - target/$TARGET/release/mortgage-calculator       # Save the compiled binary\n    expire_in: 1 week                                    # Keep artifacts for 1 week\n  allow_failure: true                                    # Allow this build to fail (cross-compilation can be tricky)\n\n# Build for macOS x86_64 (cross-compilation)\nbuild-macos:\n  extends: .build-template                               # Use build template configuration\n  variables:\n    TARGET: x86_64-apple-darwin                          # macOS 64-bit target\n  artifacts:\n    paths:\n      - target/$TARGET/release/mortgage-calculator       # Save the compiled binary\n    expire_in: 1 week                                    # Keep artifacts for 1 week\n  allow_failure: true                                    # Allow this build to fail (cross-compilation can be tricky)\n```\n\nThis GitLab CI configuration defines three build jobs that cross-compile a Rust mortgage calculator\napplication for different platforms:\n\n* `build-linux` creates a Linux x86_64 binary (required to pass)\n* `build-windows` creates Windows binaries (allowed to fail)\n* `build-macos` creates macOS x86_64 binaries (allowed to fail)\n\nAll builds use shared templates for dependency caching and consistent build environments.\n\n### Testing\n\nGitLab CI/CD streamlines code testing through its integrated pipeline system that automatically\ntriggers test suites whenever code is pushed to the repository. Developers can define multiple\ntypes of tests — unit tests, integration tests, linting, and formatting checks — all within a single\n`.gitlab-ci.yml` configuration file, with each test running in isolated Docker containers to ensure\nconsistent environments.\n\n```yaml\n# Run unit tests\ntest:unit:\n  extends: .rust-template                                # Use Rust template configuration\n  stage: test                                            # Execute during test stage\n  script:\n    - cargo test --verbose                               # Run all unit tests with verbose output\n\n# Run integration tests using the compiled binary\ntest:integration:\n  extends: .rust-template                                # Use Rust template configuration\n  stage: test                                            # Execute during test stage\n  script:\n    # Test the compiled binary with sample inputs and verify expected output\n    - target/x86_64-unknown-linux-gnu/release/mortgage-calculator --cli calculate --property-value 350000 --down-payment 70000 --interest-rate 5.0 | grep -q \"TOTAL MONTHLY PAYMENT\"\n  needs:\n    - build-linux                                        # Depends on Linux build job completing\n\n# Run Clippy linter for code quality checks\ntest:clippy:\n  extends: .rust-template                                # Use Rust template configuration\n  stage: test                                            # Execute during test stage\n  script:\n    - rustup component add clippy                        # Install Clippy linter\n    - cargo clippy -- -D warnings                       # Run Clippy and treat warnings as errors\n  allow_failure: true                                    # Allow linting failures (can be improved over time)\n\n# Check code formatting\ntest:format:\n  extends: .rust-template                                # Use Rust template configuration\n  stage: test                                            # Execute during test stage\n  script:\n    - rustup component add rustfmt                       # Install Rust formatter\n    - cargo fmt -- --check                              # Check if code is properly formatted\n  allow_failure: true                                    # Allow formatting failures (can be improved over time)\n```\n\nThis GitLab CI configuration creates four test jobs that validate a Rust mortgage calculator application:\n\n* `test:unit` runs unit tests\n* `test:integration` executes the compiled Linux binary with sample inputs to verify functionality\n* `test:clippy` performs code quality linting (allowed to fail)\n* `test:format` checks code formatting compliance (allowed to fail)\n\n## Package and Container Registries\n\nGitLab's [Package Registry](https://docs.gitlab.com/user/packages/package_registry/) provides a secure solution to the common challenge of sharing\ninternal libraries and proprietary code within organizations. This capability is essential\nfor enterprises and teams that need to maintain artifacts while leveraging the broader\nRust ecosystem.\n\nThe registry supports [generic artifacts](https://docs.gitlab.com/user/packages/generic_packages/) with fine-grained access controls that\nalign with GitLab's project permissions. This means teams can share libraries securely across\nprojects while maintaining intellectual property protection and compliance requirements.\n\nAdditonally, we can containerize our application and store the container images in\nGitLab's built-in [Container Registry](https://docs.gitlab.com/user/packages/container_registry/).\n\n### Publishing to GitLab Package Registry\n\nThis section of our `.gitlab-ci.yml` demonstrates how to package and publish Rust applications\nas tar archives to GitLab's generic package registry using CI/CD automation.\n\n```yaml\n# Package application as tar archive\npackage:tar:\n  image: alpine/curl:8.12.1                             # Lightweight image with curl for uploading\n  stage: package                                         # Execute during package stage\n  variables:\n    PACKAGE_NAME: mortgage-calculator.tar.gz             # Name of the archive file\n  script:\n    # Create tar archive of the Linux binary\n    - tar -czvf $PACKAGE_NAME target/x86_64-unknown-linux-gnu/release/mortgage-calculator\n    # Upload archive to GitLab Package Registry using API\n    - |\n      curl -v --location --header \"JOB-TOKEN: $CI_JOB_TOKEN\" \\\n      --upload-file $PACKAGE_NAME \\\n      \"$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/tar/$CI_COMMIT_BRANCH/$PACKAGE_NAME\"\n  artifacts:\n    paths:\n      - target/x86_64-unknown-linux-gnu/release/mortgage-calculator  # Save binary\n      - mortgage-calculator.tar.gz                      # Save archive\n    expire_in: 1 week                                    # Keep artifacts for 1 week\n  needs:\n    - build-linux                                        # Depends on Linux build completing\n```\n\nThis GitLab CI configuration defines one packaging job `package:tar` that creates a compressed tar archive\nof the Linux mortgage calculator binary and uploads it to GitLab's Package Registry, while also saving\nboth the binary and archive as pipeline artifacts.\n\n![Package Registry](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314541/uqnejcipnge3r1dngotm.png)\n\n### Publishing to GitLab Container Registry\n\nThe below shows the process of building Dockerfiles and publishing Docker images to GitLab's\nContainer Registry with proper tagging and authentication.\n\n```yaml\n# Package application as Docker image\npackage:docker:\n  image: docker:24.0                                     # Use Docker image for building containers\n  stage: package                                         # Execute during package stage\n  services:\n    - docker:24.0-dind                                   # Docker-in-Docker service for building images\n  before_script:\n    # Login to GitLab Container Registry\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n  script:\n    - docker build -t $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG .  # Build Docker image with commit SHA tag\n    - docker tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG $DOCKER_IMAGE_NAME:latest  # Also tag as latest\n    - docker push $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG  # Push tagged image to registry\n    - docker push $DOCKER_IMAGE_NAME:latest             # Push latest image to registry\n```\n\nThis GitLab CI configuration defines one Docker packaging job `package:docker` that builds a Docker\nimage of the mortgage calculator application, tags it with both the commit SHA and \"latest,\" and\nthen pushes both tagged versions to  GitLab's Container Registry.\n\n![Container Registry](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314537/nlodhvdnpqccf0cryfqu.png)\n\n## Security scanning\n\nGitLab security scanning provides comprehensive protection that goes beyond Rust's built-in memory\nsafety guarantees. While Rust prevents many common security vulnerabilities at compile time,\napplications still need protection against dependency vulnerabilities, unsafe code blocks, and\nlogical security issues.\n\nThe platform's Static Application Security Testing (SAST) integrates seamlessly with\nRust's toolchain, providing automated security analysis as part of the CI/CD pipeline\nThis proactive approach catches security issues before they reach production, supporting\nboth compliance requirements and secure development practices.\n\nGitLab's comprehensive security features including SAST, dependency scanning, secret detection\nand more can easily be implemented via templates, as seen below. **Note:** Additional configuration is\nrequired to [enable SAST for Rust](https://docs.gitlab.com/user/application_security/sast/#scan-a-rust-application).\n\n```yaml\n# Include GitLab's security scanning templates for DevSecOps\ninclude:\n  - template: Jobs/SAST.gitlab-ci.yml                    # Static Application Security Testing\n  - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml  # Scan dependencies for vulnerabilities\n  - template: Jobs/Container-Scanning.gitlab-ci.yml      # Scan Docker containers for vulnerabilities\n  - template: Jobs/SAST-IaC.gitlab-ci.yml               # Infrastructure as Code security scanning\n  - template: Jobs/Secret-Detection.gitlab-ci.yml        # Detect secrets in source code\n```\n\nSecurity scanners can be configured similar to how you would configure any GitLab job:\n\n```yaml\n# Configure Semgrep SAST scanning for Rust files\nsemgrep-sast:\n  rules:\n    - if: $CI_COMMIT_BRANCH                              # Run on any branch\n      exists:\n        - \"**/*.rs\"                                      # Only if Rust files exist\n  variables:\n    SAST_EXCLUDED_PATHS: \".cargo/**\"                     # Exclude Cargo cache from scanning\n\n# Scan Docker container for security vulnerabilities\ncontainer_scanning:\n  stage: container-security                              # Execute during container-security stage\n  variables:\n    CS_IMAGE: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG      # Image to scan\n    CS_DOCKERFILE_PATH: Dockerfile                       # Path to Dockerfile for context\n  needs:\n    - package:docker                                     # Depends on Docker image being built\n```\n\nWhen vulnerabilites are detected in a merge request (MR), you can see all the vulnerabilites detected\nand use the provided information to either resolve or dismiss vulnerabilities.\n\n![Vulnerability MR view](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314544/gcicke3ltvbcv57mr8zr.png)\n\nYou also can add [Security Policies](http://docs.gitlab.com/user/application_security/policies/) to require approval before vulnerable code can be merged, or to force scanners to run regardless of what is in the `.gitlab-ci.yml`.\n\n![Merge Request Approval Policy](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314538/c95nwocol03lonrr6r4n.png)\n\nYou can triage all the vulnerabilities found in your default branch by using the\n[Vulnerability Report](https://docs.gitlab.com/user/application_security/vulnerability_report/):\n\n![Vulnerability Report](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314544/b0nctigbv1ddpzizkp9x.png)\n\n## Documentation with GitLab Pages\n\n[GitLab Pages](https://docs.gitlab.com/user/project/pages/) provides an excellent platform for\nhosting Rust documentation, integrating seamlessly with Cargo's built-in documentation generation.\nThis creates a powerful workflow where API documentation, project guides, and examples are automatically\ngenerated and deployed with every code change.\n\nThe combination of `cargo doc` and GitLab Pages enables teams to maintain up-to-date documentation\nwithout manual intervention, ensuring that documentation stays synchronized with code changes.\nThis is particularly valuable for Rust projects where comprehensive documentation is essential\nto understand complex APIs and safety contracts.\n\n### Automated documentation deployment\n\nThe following code shows the CI/CD configuration for automatically generating and deploying Rust documentation using\n`cargo doc` and GitLab Pages.\n\n```yaml\n# Generate and publish documentation using GitLab Pages\nbuild-documentation:\n  extends: .rust-template                                # Use Rust template configuration\n  stage: build                                           # Execute during build stage\n  variables:\n    GIT_SUBMODULE_STRATEGY: recursive                    # Clone submodules recursively if needed\n  pages: true                                            # Enable GitLab Pages deployment\n  script:\n    - cargo doc --no-deps                                # Generate documentation without dependencies\n    - mv target/doc public                               # Move docs to public directory for Pages\n  artifacts:\n    paths:\n      - public                                           # GitLab Pages serves from public directory\n  rules:\n    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH     # Only run on default branch (main/master)\n  environment:\n    name: documentation                                  # Environment name for tracking\n    url: $CI_PAGES_URL/mortgage_calculator/index.html   # Documentation URL\n  allow_failure: true                                    # Allow documentation build to fail\n```\n\nOnce the job is complete, you can see the deployed documentation by visiting the [GitLab Environment](https://docs.gitlab.com/ci/environments/) it has been deployed to.\n\n![Pages environment](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314541/aofa6wwjugeyeshuwg9r.png)\n\nThis allows you to manage multiple versions of the documentation in different envrionments.\nThe documentation will be deployed consistent with the `cargo doc` output:\n\n![Pages build](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314541/go0tmljjzoxq5bujsdbv.png)\n\n## Deploy anywhere\n\nOne of GitLab's greatest strengths is its infrastructure-agnostic approach to deployment.\nWhether your organization runs on traditional on-premises servers, modern cloud platforms,\nhybrid environments, or edge computing infrastructure, GitLab's CI/CD system can deploy Rust\napplications seamlessly across any target environment.\n\nGitLab's deployment flexibility stems from its container-first approach and extensive integration\necosystem. The platform supports deployment to virtually any infrastructure that can run\ncontainers, virtual machines, or bare-metal applications. This versatility is particularly\nvaluable for Rust applications, which often need to run in diverse environments ranging from\nresource-constrained embedded systems to high-performance cloud clusters.\n\n### Kubernetes deployment\n\nGitLab simplifies Kubernetes deployments by providing built-in cluster integration and\npre-configured Docker images that include essential tools like Helm and kubectl, eliminating\nthe need for developers to set up complex deployment environments.\n\n```yaml\n# Deploy application to Kubernetes cluster\ndeploy:kubernetes:\n  stage: deploy                                          # Execute during deploy stage\n  image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image:helm-3.10.0-kube-1.24.6-alpine-3.15  # Image with Helm and kubectl\n  variables:\n    HELM_HOST: \"localhost:44134\"                         # Helm host configuration\n    HELM_DEPLOY_NAME: mortgage-calc-$CI_COMMIT_REF_NAME  # Deployment name based on branch\n    HELM_DEPLOY_NAMESPACE: calc-app                      # Kubernetes namespace for deployment\n    KUBE_CONTEXT: $CI_PROJECT_PATH:rust-mortgage-calculator  # Kubernetes context to use\n  script:\n    - kubectl config use-context $KUBE_CONTEXT          # Set the kubectl context\n    # Deploy using Helm with custom values and Docker image\n    - helm upgrade --install $HELM_DEPLOY_NAME chart -f chart/values.yaml\n      --namespace $HELM_DEPLOY_NAMESPACE\n      --create-namespace\n      --set image=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG\n      --set calc.name=$HELM_DEPLOY_NAME \n  needs:\n    - package:docker                                     # Depends on Docker image being available\n```\n\nThis GitLab CI configuration defines one deployment job `deploy:kubernetes` that uses Helm to deploy the\nmortgage calculator application to a Kubernetes cluster, creating or upgrading the deployment in a\ndedicated namespace while using the Docker image built in the previous packaging stage.\n\n![Kubernetes output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314538/bgmbh4qyfxcnnlqvsitc.png)\n\n## GitLab Duo AI features\n\n[GitLab Duo](https://about.gitlab.com/gitlab-duo/) AI features provide significant advantages for Rust development by offering intelligent code\nsuggestions and explanations specifically tailored to the language's unique syntax and patterns.\n\nThe GitLab platform supports Rust as one of its [directly-supported languages](https://docs.gitlab.com/user/project/repository/code_suggestions/supported_extensions/#supported-languages-by-ide) for every IDE, ensuring high-quality code completion and generation\nthat understands Rust's ownership model, memory safety principles, and idiomatic patterns. \n\n### GitLab Duo Code Suggestions\n\nGitLab Duo's ability to provide contextual code suggestions while typing helps developers navigate Rust's\nsometimes complex syntax more efficiently, reducing the learning curve for newcomers and accelerating\nproductivity for experienced developers.\n\n![Code Suggestions](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314538/uvy6hmzvyd0mnqeic9tq.png)\n\n### GitLab Duo Chat\n\n[GitLab Duo Chat](https://about.gitlab.com/blog/gitlab-duo-chat-gets-agentic-ai-makeover/) complements the code suggestions by offering conversational assistance for explaining\nRust code sections, debugging compiler errors, and providing guidance on best practices. This is\nparticularly valuable in Rust development where compiler error messages, while helpful, can\nsometimes be overwhelming for developers transitioning from other languages. The AI can help interpret\nRust's detailed error messages and suggest fixes, making the development process more efficient by\nreducing the time spent deciphering compilation issues.\n\n![GitLab Duo Chat](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756314537/depxztu1h89bez3ylwk3.png)\n\nGitLab Duo Chat can also be used directly from Vulnerability Report to explain a vulnerability. GitLab Duo [Vulnerability Explanation](https://about.gitlab.com/the-source/ai/understand-and-resolve-vulnerabilities-with-ai-powered-gitlab-duo/) represents a\nsignificant advancement in making application security more accessible and actionable for development teams. Rather than simply flagging potential issues with\ncryptic error codes. or technical jargon, AI breaks down each vulnerability's nature, potential impact, and remediation steps in terms that developers at\nall skill levels can quickly grasp. This democratization of security knowledge accelerates the remediation process, reduces the back-and-forth between security\nand development teams, and ultimately helps organizations ship more secure code faster:\n\n![Vulnerability Explain 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405091/rrcenbfazhhulmrp99yx.png)\n\u003Cp>\u003C/p>\n\n![Vulnerability Explain 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405094/b3o4lkexyn9lp41ib8ye.png)\n\u003Cp>\u003C/p>\n\n![Vulnerability Explain 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405095/y56wq8j5tg10t4dgbgfq.png)\n\u003Cp>\u003C/p>\n\n![Vulnerability Explain 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405090/mpc1mst4ydijpqdtlljm.png)\n\u003Cp>\u003C/p>\n\nGitLab Duo also provides [Agentic Chat](https://docs.gitlab.com/user/gitlab_duo_chat/agentic_chat/), which serves as an intelligent development companion for Rust\napplications, offering context-aware assistance throughout the entire development lifecycle. Developers\ncan leverage its conversational interface to generate Rust code snippets, scaffold new Rust projects with\nappropriate `Cargo.toml` configurations, and much more.\n\n### GitLab Duo Vulnerability Resolution\n\nGitLab Duo [Vulnerability Resolution](https://docs.gitlab.com/user/application_security/vulnerabilities/#vulnerability-resolution) uses AI to automatically generate specific code fixes for detected\nsecurity issues, dramatically reducing remediation time from hours to minutes. AI analyzes vulnerable code patterns\nand proposes precise patches tailored to the project's context, language, and dependencies while maintaining code\nfunctionality and style consistency. This automation is particularly effective for common vulnerabilities like SQL\ninjection and cross-site scripting, enabling development teams to maintain velocity while significantly improving their\nsecurity posture without disrupting the development workflow.\n\n![Duo Remediate Example 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405090/blpwclp68igekkecbyna.png)\n\u003Cp>\u003C/p>\n\n![Duo Remediate Example 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405090/avvncsspwyirtk14jdbe.png)\n\u003Cp>\u003C/p>\n\n### GitLab Duo Code Review\n\nGitLab Duo's [AI-powered code review](https://docs.gitlab.com/user/project/merge_requests/duo_in_merge_requests/#have-gitlab-duo-review-your-code) enhances the development process by providing intelligent, automated feedback on\nMRs before human reviewers engage. AI analyzes code changes for potential bugs, security vulnerabilities,\nperformance issues, and adherence to coding standards, offering contextual suggestions and explanations that help developers\ncatch issues early. By augmenting traditional peer reviews with consistent, immediate AI insights, this feature reduces\nthe burden on senior developers, accelerates the review cycle, and ensures that basic quality checks are consistently\napplied across all code contributions, ultimately improving code quality while allowing human reviewers to focus on\nhigher-level architectural and business logic concerns.\n\n![Duo Code Review 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405090/hewxrp2f22mf2fe4daaa.png)\n\u003Cp>\u003C/p>\n\n![Duo Code Review 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1756405091/qbw1gi0l4ngysnyjwoy8.png)\n\u003Cp>\u003C/p>\n\nThese are just some of the AI feautures that can be used to allow you to ship more secure Rust software faster than ever.\nTo learn about all the GitLab AI features provided, visit the [GitLab Duo solution page](https://about.gitlab.com/gitlab-duo/).\n\n## The GitLab advantage for Rust\n\nGitLab provides a complete development platform that matches Rust's comprehensive approach:\n\n**Integrated Workflow:**\n- **Single Platform:** Code, CI/CD, security, and deployment in one place\n- **Rust-optimized:** Docker-based builds perfect for Rust's toolchain\n- **Security First:** Built-in security scanning\n- **Enterprise-ready:** Scalable infrastructure for large teams\n\n**Performance Benefits:**\n- **Efficient Caching:** Speeds up Rust's longer compilation times\n- **Parallel Builds:** Maximizes GitLab Runner efficiency\n- **Artifact Management:** Streamlined binary distribution\n\n**Developer Experience:**\n- **Familiar Tools:** Leverage standard Rust tooling (Cargo, Clippy, rustfmt)\n- **Visual Feedback:** Comprehensive dashboards and reporting\n- **Automation:** Reduces manual deployment and testing overhead\n- **GitLab Duo AI:** Ship more secure software faster with AI throughout the entire software development lifecycle\n\nGitLab's platform capabilities perfectly complement Rust's strengths, creating an ecosystem\nwhere safety, performance, and developer productivity converge. Rust applications on GitLab\nrepresent the cutting edge of software development—powered by a platform that understands and\nenhances the Rust development experience.\n\nTo learn more about the benefits of GitLab, sign up for a [free trial of GitLab Ultimate with Duo Enterprise](https://about.gitlab.com/free-trial/).\n","content:en-us:blog:secure-rust-development-with-gitlab.yml","Secure Rust Development With Gitlab","en-us/blog/secure-rust-development-with-gitlab.yml","en-us/blog/secure-rust-development-with-gitlab",{"_path":3883,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3884,"content":3889,"config":3895,"_id":3897,"_type":13,"title":3898,"_source":15,"_file":3899,"_stem":3900,"_extension":18},"/en-us/blog/securing-your-code-on-gitlab",{"title":3885,"description":3886,"ogTitle":3885,"ogDescription":3886,"noIndex":6,"ogImage":3100,"ogUrl":3887,"ogSiteName":685,"ogType":686,"canonicalUrls":3887,"schema":3888},"The ultimate guide to securing your code on GitLab.com","This in-depth tutorial, complete with best practices, will help you secure your development environment.","https://about.gitlab.com/blog/securing-your-code-on-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The ultimate guide to securing your code on GitLab.com\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Steve Grossman\"}],\n        \"datePublished\": \"2023-05-31\",\n      }",{"title":3885,"description":3886,"authors":3890,"heroImage":3100,"date":3892,"body":3893,"category":830,"tags":3894},[3891],"Steve Grossman","2023-05-31","\n\nA critical aspect of a DevSecOps methodology is to apply best practices to secure your development environment. Your software should be protected from malicious and accidental exposure or modification. This blog explains how to control and manage access to GitLab.com and, in turn, source code, build pipelines, dependency and package repositories, and deployment keys involved in the [software supply chain](/blog/the-ultimate-guide-to-software-supply-chain-security/). The best practices specifically address the capabilities for end users on multi-tenant GitLab.com and are written for the Ultimate license tier. Not all of these capabilities are available at the Premium tier.\n\n## 1. Group settings\nMany security-related settings can be set on the top-level group and will cascade down into all subgroups and projects. They are the easiest and most important in securing your GitLab.com instance.\n\n### General settings\nIn the top-level group, the following settings should be applied to provide the best security for the code within that group:\n\n#### Make the group visibility level private\nThis is likely the most important setting among general settings. By marking the group “private\", anyone who is not explicitly a member of the group will not be able to access it. Additionally, by making the top-level group private, all subgroups and projects will also be private and cannot be exposed.  \n\n#### Permissions and group features\nUnder permissions:\n - Set “Prevent members from sending invitations to outside groups”. This will prevent accidentally adding people who should not belong to the group.\n - Set “Prevent sharing a project with other groups”. This prevents accidental or malicious exfiltration of code by sharing or moving a project to another group outside the control of the top-level group owner.\n - Allow project and group access token creation. Project and group access tokens are much like [personal access tokens](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with the following improvements:\n    - They are visible to and manageable by group owners and maintainers, which means they can be revoked and have expiration dates set by an administrator to limit the opportunity for abuse.\n    - They create a virtual “bot” user that does not count against your license count.\n - Enable [delayed project deletion](/blog/delayed-deletion/). This will give you a seven-day grace period to catch and prevent accidental or malicious removal of a repo. GitLab.com, like self-managed GitLab, does not have the ability to restore an individual project without significant expense for professional services.\n - Allowlist the Classless Inter-Domain Routing (CIDR) or supernetting from which users should be accessing the code.\n - Restrict membership to only those email domains belonging to your organization and contractors.\n - Restrict creation of subgroups to Owners. This will help keep the structure of the top-level group within your policies and make [SAML Group Sync](https://docs.gitlab.com/ee/user/group/saml_sso/group_sync.html) for membership easier to manage.\n - Block forking projects outside of this group hierarchy. This will help prevent code exfiltration.\n - Require [two-factor authentication](https://docs.gitlab.com/ee/user/profile/account/two_factor_authentication.html). This disables the ability to use password authentication with Git over HTTPS.\n - Disallow adding new members to projects within this group. All members must be inherited from the group.  \n\n#### Merge request approvals\n[Merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/) help prevent injection of malicious code into the repository by having people other than the author review them. Enable merge request approvals for all projects in your group to:  \n - Prevent approval by authors.\n - Prevent approvals by users who add commits.\n - Prevent editing approval rules in projects and on individual merge requests.  \n\n### SAML SSO\nTo more tightly control who can access your code in GitLab.com, set up [SAML SSO](https://docs.gitlab.com/ee/user/group/saml_sso/). This will ensure that everyone who accesses it is approved by someone in authority.\n\n#### To configure SAML SSO:  \n - Enable SAML authentication for this group.\n - Enforce SSO-only authentication for web activity for this group.\n - Enforce SSO-only authentication for Git and Dependency Proxy activity for this group.\n - Set the Default membership role to Minimal Access. Roles can be increased as needed in subgroups or individual projects, minimal access prevents any visibility to projects or subgroups where the user is not explicitly granted another role.\n - Tightly control access to the Maintainer and Owner roles; every developer does not need to have a Maintainer role.\n\n## 2. Group auditing and compliance\nRegularly and periodically review the [compliance reports](https://docs.gitlab.com/ee/user/compliance/compliance_report/) to verify who is approving merge requests and what MRs are getting approved.\n\nSet up streaming group audit events to your corporate security information and event management (SIEM) system and monitor them for unusual activity. This needs to be repeated for each group and project in the hierarchy to get the maximum number of audit events.\n\n## 3. Group-level push rules\n Setting restrictive push rules at the group level will help ensure malicious code is not injected into the repository:\n - Require committers be verified.\n - Reject unsigned commits.\n - Ensure the commit author is a GitLab user.\n - Prevent pushing secret files.\n - Require commit author’s email to be from your email domain.  \n\n## CI/CD  \nThe following settings can help insure the integrity of [CI/CD](/topics/ci-cd/) pipelines and reduce the opportunities for abuse and malice:\n - Register runners at the lowest practical level to reduce the blast radius of any malicious use.\n - Require tags to use all runners to reduce the opportunity for abuse.\n - Define CI/CD variables – especially if they contain secrets – at the lowest practical level to reduce the blast radius of any malicious use.\n - Use protected runners with protected variables and protected branches to significantly limit who can deploy into production environments or misuse cloud resources.\n - Access to change the .gitlab-ci.yml pipeline definition file should be tightly controlled in all repos through the CODEOWNERS file to prevent malicious use of the CI/CD system.\n\n## 4. Project settings  \nSome settings do not cascade down from the group or are not available at the group level and must be set on individual projects instead. These include some repo-specific settings.\n\n### Repository\nSet up protected branches and protected tags to go along with the protected runners and protected variables defined above.\n\n### CI/CD  \n#### General \n - Disable public pipelines.\n - Use separate caches for protected branches.  \n\n### Protected environments\nUse protected environments and tightly limit who can deploy and require approvals for deploying.\n\n#### Token access\nRestrict access to this project’s CI_JOB_TOKEN to only individual projects to ensure malicious projects to not retrieve the token and use it to access the API.\n\n#### Secure files\nStore keystores, provisioning profiles and signing certificates in the Secure Files storage rather than the repository.  \n\n## 5. Project-level security testing and compliance\n### Configuration\n#### Security testing\n - Enable static application security testing [SAST](https://docs.gitlab.com/ee/user/application_security/sast/) to help prevent insertion of malicious code into the application.\n - Enable dependency scanning and regularly review the dependency list or software, or software bill of materials ([SBOM](/blog/the-ultimate-guide-to-sboms/)], generated by dependency scanning for vulnerabilities and malicious components.\n - Enable [container scanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/) and cluster image scanning.  \n\n#### Policies\nAs an alternative to the security testing section above, you may choose to enable scan execution policies.\nEnable [test scan result policies](https://docs.gitlab.com/ee/user/application_security/policies/scan-result-policies.html) to prevent merging code with critical vulnerabilities.\n\n\nFollowing these best practices will help ensure that your code hosted on GitLab.com is safe from tampering and \n[public exposure](https://www.engadget.com/okta-stolen-source-code-205601214.html) and that your \nsoftware supply chain is secure and only authorized users \nare accessing your software assets.\n\n## More resources\n- [Group level settings documentation](https://docs.gitlab.com/ee/user/group/)\n- [Project level settings documentation](https://docs.gitlab.com/ee/user/project/settings/)\n",[9,495],{"slug":3896,"featured":6,"template":700},"securing-your-code-on-gitlab","content:en-us:blog:securing-your-code-on-gitlab.yml","Securing Your Code On Gitlab","en-us/blog/securing-your-code-on-gitlab.yml","en-us/blog/securing-your-code-on-gitlab",{"_path":3902,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3903,"content":3908,"config":3915,"_id":3917,"_type":13,"title":3918,"_source":15,"_file":3919,"_stem":3920,"_extension":18},"/en-us/blog/self-service-security-alert-handling-with-gitlabs-uam",{"title":3904,"description":3905,"ogTitle":3904,"ogDescription":3905,"noIndex":6,"ogImage":3831,"ogUrl":3906,"ogSiteName":685,"ogType":686,"canonicalUrls":3906,"schema":3907},"Self-service security alert handling with GitLab's UAM","The User Attestation Module automates security alerts by routing them directly to team members for verification, reducing manual SecOps work and enhancing audit trails.","https://about.gitlab.com/blog/self-service-security-alert-handling-with-gitlabs-uam","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Self-service security alert handling with GitLab's UAM\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Bala Allam\"},{\"@type\":\"Person\",\"name\":\"Matt Coons\"}],\n        \"datePublished\": \"2025-03-18\",\n      }",{"title":3904,"description":3905,"authors":3909,"heroImage":3831,"date":3912,"body":3913,"category":830,"tags":3914},[3910,3911],"Bala Allam","Matt Coons","2025-03-18","The [GitLab Security Operations team](https://handbook.gitlab.com/handbook/security/security-operations/) prioritizes automation that enables security engineers to focus on high-impact work rather than routine tasks that can be automated. A key innovation in this automation strategy is creation of the User Attestation Module (UAM), which allows GitLab team members to directly respond to and verify security alerts flagged as potentially malicious. When the [GUARD framework](https://about.gitlab.com/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd/) detects suspicious activity, it routes the alert to the relevant team member for review. The team member can then attest whether they recognize and authorize the activity. Their response is recorded for audit purposes, and, based on their input, the system either closes the alert or escalates it to the Security Incident Response Team (SIRT).\n\nIn this article, you'll learn about the UAM and how it can benefit your DevSecOps environment.\n\n## How the User Attestation Module works\n\nThe UAM streamlines security alert handling through a comprehensive workflow that includes:\n\n* Alert verification by team members  \n* Collection and documentation of supporting evidence  \n* Option to request additional support from GitLab SecOps  \n* Secure storage of team member responses  \n* Automated alert resolution or incident escalation  \n* Team member feedback collection for continuous improvement\n\nWe created UAM to help us: \n\n1. Route low priority alerts (such as administrative activities) to the relevant team members who performed them.  \n2. Reduce alert fatigue by first checking with the team member who completed the activity before routing to SIRT if necessary.   \n3. Collect and store team member responses to maintain an audit trail and rich metrics.   \n4. Create a response tier between **SIRT needs to triage this alert** and **This is an informational signal that does not need to be reviewed directly**.\n\n## UAM's design principles\n\nThe UAM is a Slack-first automation that reaches out to team members to validate activity directly in Slack, reducing effort and increasing participation. Today, 40% of all security alerts are delivered to team members through the UAM, saving SIRT valuable time to focus on higher importance alerts and incidents. \n\nA robust escalation workflow in the UAM ensures that all alerts are validated by team members or escalated to SIRT. When a UAM alert reaches a team member, they have a period of time to respond attesting to the activity or stating they do not recognize the activity. If no response is recorded, the UAM alert is auto-escalated to SIRT for handling. \n\nComprehensive metrics collection is a core GUARD design principle, which extends to how we designed UAM. All user interactivity with triggered UAM alerts is logged in a metrics database, which enables comprehensive measurement to identify problematic alerts, opportunities for process improvement, and overall UAM health.\n\nUAM enables a third alert tier, bridging the gap between alerts that always needed to be investigated, and lower importance informational signals that are grouped by entity for escalation and correlation. \n\n- Stable alerts (must be triaged and investigated by SIRT)  \n- UAM alerts (routed to team members to attest to the activity)  \n- Informational signal (low-importance events that are interesting and correlated by entity grouping)\n\n## UAM components \n\nThe UAM framework consists of multiple components:\n\n- GitLab: Fetches a user email address based on user_id via user’s API and stores user's responses  - Slack: Searches each user by email using Slack API and posts a UAM notification to the end user as well as collects responses from users using Slack modals  \n- Tines: Processes and orchestrates user responses and alerts     \n- Devo: Receives alert payload and alert notifications  \n- Metrics DB: Records metrics for triggered UAM alerts\n\nThe workflow integrates with following modules:\n\n- GitLab API for user identification  \n- Slack API for user communication  \n- Webhook configuration for alert reception  \n- Audit trail storage in GitLab\n\n## UAM workflow\n\nThe diagram below illustrates the workflow of the UAM module:\n\n![UAM - flow chart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674678/Blog/Content%20Images/UAM_detection_edited.png)\n\n## Following along with GUARD\n\nWe are still unveiling parts of GUARD and how it works, so [follow along](https://gitlab.com/gitlab-security-oss/guard) to learn how we automate our security detections from end to end.\n\n## Read more about the GUARD framework\n\n- [Unveiling the GUARD framework to automate security detections at GitLab](https://about.gitlab.com/blog/unveiling-the-guard-framework-to-automate-security-detections-at-gitlab/)  \n- [Automating cybersecurity threat detections with GitLab CI/CD](https://about.gitlab.com/blog/automating-cybersecurity-threat-detections-with-gitlab-ci-cd/)\n- [Open Source Security at GitLab](https://about.gitlab.com/security/open-source-resources/)",[830,9,495,694,743],{"slug":3916,"featured":6,"template":700},"self-service-security-alert-handling-with-gitlabs-uam","content:en-us:blog:self-service-security-alert-handling-with-gitlabs-uam.yml","Self Service Security Alert Handling With Gitlabs Uam","en-us/blog/self-service-security-alert-handling-with-gitlabs-uam.yml","en-us/blog/self-service-security-alert-handling-with-gitlabs-uam",{"_path":3922,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3923,"content":3929,"config":3935,"_id":3937,"_type":13,"title":3938,"_source":15,"_file":3939,"_stem":3940,"_extension":18},"/en-us/blog/set-up-flux-for-gitops-on-openshift",{"title":3924,"description":3925,"ogTitle":3924,"ogDescription":3925,"noIndex":6,"ogImage":3926,"ogUrl":3927,"ogSiteName":685,"ogType":686,"canonicalUrls":3927,"schema":3928},"Set up Flux for GitOps to deploy workloads on OpenShift","Learn how to set up a sample project, complete a bootstrap Flux installation, and authenticate your installation with a project deploy token.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682825/Blog/Hero%20Images/genericworkflow.jpg","https://about.gitlab.com/blog/set-up-flux-for-gitops-on-openshift","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Set up Flux for GitOps to deploy workloads on OpenShift\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Bart Zhang\"}],\n        \"datePublished\": \"2023-07-05\",\n      }",{"title":3924,"description":3925,"authors":3930,"heroImage":3926,"date":3932,"body":3933,"category":785,"tags":3934},[3931],"Bart Zhang","2023-07-05","In February, we announced that [Flux CD would be our recommended approach to\ndo GitOps with\nGitLab](https://about.gitlab.com/blog/why-did-we-choose-to-integrate-fluxcd-with-gitlab/).\nThis tutorial explains how to set up GitLab and Flux to deploy workloads on\nRed Hat OpenShift. You’ll set up a sample project, complete a bootstrap Flux\ninstallation, and authenticate your installation with a project deploy\ntoken. By the end of this tutorial, you should be able to deploy an example\nNGINX workload to OpenShift from a GitLab Repo via Flux.\n\n\nYou can find the fully configured tutorial project in [this GitLab\nrepository](https://gitlab.com/gitlab-partner-demos/red-hat-demos/flux). It\nworks in conjunction with [this\nrepository](https://gitlab.com/gitlab-partner-demos/red-hat-demos/web-app-manifests),\nwhich contains the example OpenShift manifest. \n\n\n### To set up Flux for GitOps:\n\n1. Create a personal access token\n\n2. Create the Flux repository\n\n3. Create the OpenShift manifest repository\n\n4. Configure Flux to sync your manifests\n\n5. Verify your configuration\n\n\n### Prerequisites:\n\nYou must have an OpenShift cluster running. Cluster-admin privileges are\nrequired to install Flux on OpenShift, which can either be installed via\nOperatorHub or the CLI.\n\n\nWhen installing Flux with CLI, you need to set the nonroot SCC for all\ncontrollers in the flux-system namespace like this:\n\n\n```\n\nNS=\"flux-system\"\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:kustomize-controller\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:helm-controller\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:source-controller\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:notification-controller\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:image-automation-controller\n\noc adm policy add-scc-to-user nonroot\nsystem:serviceaccount:${NS}:image-reflector-controller\n\n```\n\nExpected output:\n\n```\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"kustomize-controller\"\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"helm-controller\"\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"source-controller\"\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"notification-controller\"\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"image-automation-controller\"\n\nclusterrole.rbac.authorization.k8s.io/system:openshift:scc:nonroot added:\n\"image-reflector-controller\"\n\n```\n\n\nAlso, you'll need to [patch your\nKustomization](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.8/html/security_and_compliance/seccomp-profiles)\nto remove the SecComp Profile and enforce runUserAs to the same UID provided\nby the images to prevent OpenShift to alter the user expected by our\ncontrollers, prior to bootstrapping the cluster.\n\n\nYou’ll need to create a Git repository and clone it locally. I chose to\ncreate [the web-app-manifests\nrepository](https://gitlab.com/gitlab-partner-demos/red-hat-demos/web-app-manifests)\nto store my manifest file once it is created through the following steps.\n\n\nCreate the file structure required by bootstrap using the following command:\n\n\n```\n\ngit clone https://gitlab.com/gitlab-partner-demos/red-hat-demos/flux/\n\ncd flux\n\nmkdir -p clusters/my-cluster/flux-system\n\ntouch clusters/my-cluster/flux-system/gotk-components.yaml \\\n    clusters/my-cluster/flux-system/gotk-sync.yaml \\\n    clusters/my-cluster/flux-system/kustomization.yaml\n```\n\n\nAdd the following YAML snippet and its patches section to\nflux/clusters/my-cluster/flux-system/kustomization.yaml:\n\n\n```\n\napiVersion: kustomize.config.k8s.io/v1beta1\n\nkind: Kustomization\n\nresources:\n  - gotk-components.yaml\n  - gotk-sync.yaml\npatches:\n  - patch: |\n      apiVersion: apps/v1\n      kind: Deployment\n      metadata:\n        name: all\n      spec:\n        template:\n          spec:\n            containers:\n              - name: manager\n                securityContext:\n                  runAsUser: 65534\n                  seccompProfile:\n                    $patch: delete      \n    target:\n      kind: Deployment\n      labelSelector: app.kubernetes.io/part-of=flux\n```\n\n\nCommit and push the changes to main branch:\n\n\n```\n\ncd ~/flux\n\ngit add -A && git commit -m \"init flux for openshift\" && git push\n\n```\n\n\n### Create a personal access token\n\n\nTo authenticate with the Flux CLI, you must create a GitLab personal access\ntoken\n([PAT](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html))\nwith the api scope:\n\n1. In the upper-right corner, select your avatar.\n\n2. Select Edit profile.\n\n3. On the left sidebar, select Access Tokens.\n\n4. Enter a name and expiry date for the token.\n\n5. Select the api scope.\n\n6. Select Create personal access token.\n\n7. Copy the new token to your clipboard.\n\n\nNote: You can also use a project or group access token with the api scope.\n\n\n### Create the Flux repository\n\nCreate a Git repository, install Flux, and authenticate Flux with your repo\nin RedHat OpenShift:\n\n1. Make sure you are logged in as an OpenShift user in your CLI to access\nyour cluster. `oc login` command is useful here.\n\n2. [Install the Flux CLI](https://fluxcd.io/flux/installation/#bootstrap).\nYou must install Flux v2 or higher. `brew install fluxcd/tap/flux` on Mac\nOSX. Check your flux version with `flux -v`. Mine is `flux version\n2.0.0-rc.1`.\n\n3. In GitLab, create a new empty project called `flux`. I chose to use [the\nrespository in this\nreadme](https://gitlab.com/gitlab-partner-demos/red-hat-demos/flux/)\n\n4. From your shell, export a GITLAB_TOKEN environment variable with the\nvalue of your personal access token. For example, `export\nGITLAB_TOKEN=\u003Cpersonal-access-token>`.\n\n5. Run the bootstrap command. The exact command depends on whether you are\ncreating the Flux repository under a GitLab user, group, or subgroup. For\nmore information, see the Flux bootstrap documentation.\n\n\nIn this tutorial, you’re working with a public project in a subgroup. The\nbootstrap command looks like this:\n\n\n```\n\ncd ~/flux\n\nflux bootstrap gitlab \\\n  --owner=gitlab-partner-demos/red-hat-demos \\\n  --repository=flux \\\n  --branch=master \\\n  --path=clusters/my-cluster \\\n  --token-auth\n```\n\nExpected output:\n\n```\n\n► connecting to https://gitlab.com\n\n► cloning branch \"master\" from Git repository\n\"https://gitlab.com/gitlab-partner-demos/red-hat-demos/flux.git\"\n\n✔ cloned repository\n\n► generating component manifests\n\n✔ generated component manifests\n\n✔ component manifests are up to date\n\n► installing components in \"flux-system\" namespace\n\n✔ installed components\n\n✔ reconciled components\n\n► determining if source secret \"flux-system/flux-system\" exists\n\n✔ source secret up to date\n\n► generating sync manifests\n\n✔ generated sync manifests\n\n✔ sync manifests are up to date\n\n► applying sync manifests\n\n✔ reconciled sync configuration\n\n◎ waiting for Kustomization \"flux-system/flux-system\" to be reconciled\n\n```\n\n\nThis command installs the Flux agent on the OpenShift cluster and configures\nit to manage itself from the repository flux-config. The command also\nautomatically creates the project deploy token required to access the\nflux-config repository.\n\n\nGreat work! You now have a repository bootstrapped with a Flux\nconfiguration. Any updates to your repository are automatically synced to\nthe cluster.\n\n\n### Create the OpenShift manifest repository\n\nNext, create a repository for your Flux manifest files. These are stateful\nfiles that track the current running configuration by\n\nthe Flux agent. I chose to use\n[web-app-manifests](https://gitlab.com/gitlab-partner-demos/red-hat-demos/web-app-manifests)\nproject to track my manifest files.\n\n1. In GitLab, create a new repository called `web-app-manifests`.\n\n1. Add a file to web-app-manifests named `nginx-deployment.yaml` with the\nfollowing contents:\n\n\n```\n\napiVersion: apps/v1\n\n\nkind: Deployment\n\n\nmetadata:\n  name: nginx-deployment\n  labels:\n    app: nginx\nspec:\n  replicas: 3\n  selector:\n    matchLabels:\n      app: nginx\n  template:\n    metadata:\n      labels:\n        app: nginx\n    spec:\n      containers:\n      - name: nginx-unprivileged\n        image: nginxinc/nginx-unprivileged:latest\n        ports:\n        - containerPort: 80\n```\n\n\nIn the new `web-app-manifests` repository, create a [GitLab deploy\ntoken](https://docs.gitlab.com/ee/user/project/deploy_tokens/) with only the\n`read_repository` scope.\n\n\nStore your deploy token username and password somewhere safe. I used\nenvironmental variables to save mine:\n\n\n```\n\nexport GITLAB_DEPLOY_TOKEN_USER=\u003Cmy-gitlab-deployment-token-username>\n\nexport GITLAB_DEPLOY_TOKEN_PASS=\u003Cmy-gitlab-deployment-token-password>\n\nenv |grep GITLAB_DEPLOY_TOKEN\n\n```\n\nExpected output:\n\n```\n\nGITLAB_DEPLOY_TOKEN_USER=myGitLabUserName\n\nGITLAB_DEPLOY_TOKEN_PASS=MySecretToken\n\n```\n\n\nIn Flux CLI, create a secret with your deploy token and point the secret to\nthe new repository. For example:\n\n\n```\n\nflux create secret git flux-deploy-authentication \\\n         --url=https://gitlab.com/gitlab-partner-demos/red-hat-demos/web-app-manifests \\\n         --namespace=default \\\n         --username=$GITLAB_DEPLOY_TOKEN_USER \\\n         --password=$GITLAB_DEPLOY_TOKEN_PASS\n```\n\nExpected output:\n\n```\n\n► git secret 'flux-deploy-authentication' created in 'default' namespace\n\n```\n\n\nTo check if your secret was generated successfully, run:\n\n\n```\n\noc -n default get secrets flux-deploy-authentication -o yaml\n\n```\n\nExpected output:\n\n```\n\napiVersion: v1\n\ndata:\n  password: Base64EncodedPassword=\n  username: Base64EncodedUsername\nkind: Secret\n\nmetadata:\n  creationTimestamp: \"2023-04-20T18:22:33Z\"\n  name: flux-deploy-authentication\n  namespace: default\n  resourceVersion: \"8168670\"\n  uid: 16292254-83cd-4df2-8a9c-bc4c718e4b4a\ntype: Opaque\n\n```\n\n\nUnder data, you should see base64-encoded values associated with your token\nusername and password.\n\n\nCongratulations! You now have a manifest repository, a deploy token, and a\nsecret generated directly on your cluster.\n\n\n### Configure Flux to sync your manifests\n\nNext, tell flux-config to sync with the web-app-manifests repository.\n\n\nTo do so, create a [GitRepository\nresource](https://docs.openshift.com/container-platform/3.11/dev_guide/application_lifecycle/new_app.html)\nin OpenShift:\n\n\n1. Clone the flux repo to your machine.\n\n```\n\n# Remember that we already have the flux repo cloned into our home dir.\n\ncd ~/flux\n\ngit pull\n\n```\n\n\n2. In your local clone of flux, add the GitRepository file\n`clusters/my-cluster/web-app-manifests-source.yaml`:\n  \n```\n",[9,549],{"slug":3936,"featured":6,"template":700},"set-up-flux-for-gitops-on-openshift","content:en-us:blog:set-up-flux-for-gitops-on-openshift.yml","Set Up Flux For Gitops On Openshift","en-us/blog/set-up-flux-for-gitops-on-openshift.yml","en-us/blog/set-up-flux-for-gitops-on-openshift",{"_path":3942,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3943,"content":3949,"config":3954,"_id":3956,"_type":13,"title":3957,"_source":15,"_file":3958,"_stem":3959,"_extension":18},"/en-us/blog/simple-kubernetes-management-with-gitlab",{"title":3944,"description":3945,"ogTitle":3944,"ogDescription":3945,"noIndex":6,"ogImage":3946,"ogUrl":3947,"ogSiteName":685,"ogType":686,"canonicalUrls":3947,"schema":3948},"Simple Kubernetes management with GitLab","Follow our tutorial to provision a Kubernetes cluster and manage it with IAC using Terraform and Helm in 20 minutes or less.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749670037/Blog/Hero%20Images/auto-deploy-google-cloud.jpg","https://about.gitlab.com/blog/simple-kubernetes-management-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Simple Kubernetes management with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2022-11-15\",\n      }",{"title":3944,"description":3945,"authors":3950,"heroImage":3946,"date":3951,"body":3952,"category":785,"tags":3953},[847],"2022-11-15","Kubernetes can be very complex and has dozens of tutorials out there on how\nto provision and manage a cluster. This tutorial aims to provide a simple,\nlightweight solution to provision a Kubernetes cluster and manage it with\ninfrastructure as code (IaC) using Terraform and Helm in 20 minutes or less.\n\n\n**The final product of this tutorial will be two IaC repositories with fully\nfunctional CI/CD pipelines:**\n\n\n1.\n[gitlab-terraform-k8s](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-eks)\n- A single source of truth to provision, configure, and manage your\nKubernetes infrastructure using Terraform\n\n1.\n[cluster-management](https://gitlab.com/gitlab-org/project-templates/cluster-management)\n- A single source of truth to define the desired state of your Kubernetes\ncluster using the GitLab Agent for Kubernetes and Helm\n\n\n![Final\nProduct](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/final-product.png){:\n.shadow}\n\n\n\n### Prerequisites\n\n- AWS or GCP account with permissions to provision resources\n\n- GitLab account \n\n- Access to a GitLab Runner\n\n- 20 minutes\n\n\n### An overview of this tutorial is as follows:\n\n\n1. Set up the GitLab Terraform Kubernetes Template 🏗️\n\n2. Register the GitLab Agent 🕵️\n\n3. Add in Cloud Credentials ☁️🔑\n\n4. Set up the Kubernetes Cluster Management Template 🚧\n\n5. Enjoy your Kubernetes Cluster completely managed in code! 👏\n\n\n## Set up the GitLab Terraform Kubernetes Template\n\n\nStart by importing the example project by URL -\n[https://gitlab.com/projects/new#import_project](https://gitlab.com/projects/new#import_project)\n\n\nTo import the project:\n\n\n1. In GitLab, on the top bar, select **Main menu > Projects > View all\nprojects**.\n\n2. On the right of the page, select **New project**.\n\n3. Select **Import project**.\n\n4. Select **Repository by URL**.\n\n5. For the Git repository URL:\n\n- [GCP Google Kubernetes\nEngine](https://cloud.google.com/kubernetes-engine):\nhttps://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-gke.git\n\n- [AWS Elastic Kubernetes Service](https://aws.amazon.com/eks/):\nhttps://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-eks.git\n\n6. Complete the fields and select **Create project**.\n\n\n## Register the GitLab Agent\n\n\nWith your newly created **gitlab-terraform-k8s** repo, create a GitLab Agent\nfor Kubernetes:\n\n\n1. On the left sidebar, select **Infrastructure > Kubernetes clusters**.\nSelect **Connect a cluster (agent).**\n\n2. From the **Select an agent** dropdown list, select **eks-agent/gke-agent\nand select **Register an agent**.\n\n3. GitLab generates a registration token for the agent. **Securely store\nthis secret token, as you will need it later.**\n\n4. GitLab provides an address for the agent server (KAS). Securely store\nthis as you will also need it later.\n\n5. Add this to the\n**gitlab-terraform-eks/.gitlab/agents/eks-agent/config.yaml** in order to\nallow the GitLab Agent to have access to your entire group.\n\n\n```yaml\n\nci_access:\n  groups:\n    - id: your-namespace-here\n```\n\n\n![Register GitLab\nAgent](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/register-gitlab-agent.png){:\n.shadow}\n\n\n\n## Add in your Cloud Credentials to CI/CD variables\n\n\n### [AWS EKS](https://aws.amazon.com/eks/)\n\n\nOn the left sidebar, select **Settings > CI/CD. Expand Variables**.\n\n1. Set the variable **AWS_ACCESS_KEY_ID** to your AWS access key ID.\n\n2. Set the variable **AWS_SECRET_ACCESS_KEY** to your AWS secret access key.\n\n3. Set the variable **TF_VAR_agent_token** to the agent token displayed in\nthe previous task.\n\n4. Set the variable **TF_VAR_kas_address** to the agent server address\ndisplayed in the previous task.\n\n\n![Add in CI/CD\nvariables](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/cicd-variables.png){:\n.shadow}\n\n\n\n### [GCP GKE](https://cloud.google.com/kubernetes-engine)\n\n\n1. To authenticate GCP with GitLab, create a GCP service account with the\nfollowing roles: **Compute Network Viewer, Kubernetes Engine Admin, Service\nAccount User, and Service Account Admin**. Both User and Admin service\naccounts are necessary. The User role impersonates the default service\naccount when creating the node pool. The Admin role creates a service\naccount in the kube-system namespace.\n\n2. **Download the JSON file** with the service account key you created in\nthe previous step.\n\n3. On your computer, encode the JSON file to base64 (replace\n/path/to/sa-key.json to the path to your key):\n\n\n```\n\nbase64 -i /path/to/sa-key.json | tr -d\n\n```\n\n\n- Use the output of this command as the **BASE64_GOOGLE_CREDENTIALS**\nenvironment variable in the next step.\n\n\nOn the left sidebar, select **Settings > CI/CD. Expand Variables**.\n\n5. Set the variable **BASE64_GOOGLE_CREDENTIALS** to the base64 encoded JSON\nfile you just created.\n\n6. Set the variable **TF_VAR_gcp_project** to your GCP’s project name.\n\n7. Set the variable **TF_VAR_agent_token** to the agent token displayed in\nthe previous task.\n\n8. Set the variable **TF_VAR_kas_address** to the agent server address\ndisplayed in the previous task.\n\n\n## Run GitLab CI to deploy your Kubernetes cluster!\n\n\n![Deploy Kubernetes\ncluster](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/pipeline-view.png){:\n.shadow}\n\n\nWhen successfully completed, view the cluster in the AWS/GCP console!\n\n\n![AWS\nEKS](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/aws-eks.png){:\n.shadow}\n\n\n### You are halfway done! 👏 Keep it up!\n\n\n## Set up the Kubernetes Cluster Management Project\n\n\nCreate a project from the cluster management project template -\n[https://gitlab.com/projects/new#create_from_template](https://gitlab.com/projects/new#create_from_template)\n\n\n1. In GitLab, on the top bar, select **Main menu > Projects > View all\nprojects**.\n\n2. On the right of the page, select **New project**.\n\n3. Select **Create from template**.\n\n4. From the list of templates, next to **GitLab Cluster Management**, select\n**Use template**.\n\n5. Enter the project details. Ensure this project is created in the same\nnamespace as the gitlab-terraform-k8s project.\n\n6. Select **Create project**.\n\n7. Once the project is created on the left sidebar, select **Settings >\nCI/CD. Expand Variables**.\n\n8. Set the variable KUBE_CONTEXT to point to the GitLab Agent. For example,\n`noah-ing-demos/infrastructure/gitlab-terraform-eks:eks-agent`.\n\n\n![Set Kube\nContext](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/kube-config.png){:\n.shadow}\n\n\n\n- **Uncomment the applications you'd like to be installed** into your\nKubernetes cluster in the **helmfile.yaml**. In this instance I chose\ningress, cert-manager, prometheus, and Vault. \n\n\n![Uncomment Applications in\nhelmfile](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/helmfile.png){:\n.shadow}\n\n\nThat will trigger your **CI/CD pipeline** and it should look like this.\n\n\n![Cluster Management\nCI/CD](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/cluster-management-cicd.png){:\n.shadow}\n\n\nOnce completed, **go to the AWS/GCP console** and check out all the deployed\nresources!\n\n\n![Deployed EKS\napplications](https://about.gitlab.com/images/blogimages/2022-11-11-simple-kubernetes-management-with-gitlab/deployed-eks-applications.png){:\n.shadow}\n\n\n### Voila! 🎉\n\n\n## Enjoy your Kubernetes cluster completely defined in code! 👏👏👏\n\n\nNow with these two repositories you can **manage a Kubernetes cluster\nentirely through code**:\n\n\n- For managing the Kubernetes cluster's infrastructure and configuring its\nresources you can make changes to the\n[gitlab-terraform-eks](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-eks)\nrepository you have setup. This project has a **Terraform CI/CD pipeline**\nthat will allow you to **review, provision, configure, and manage your\nKubernetes** infrastructure with ease.\n\n\n- For managing the desired state of the Kubernetes cluster, the\n[cluster-management](https://gitlab.com/gitlab-org/project-templates/cluster-management)\nrepository has a **GitLab Agent** set up and will **deploy any Kubernetes\nobjects defined in the helm files**.\n\n\n➡️ Bonus: If you'd like to deploy your own application to the Kubernetes\ncluster, then add to your **cluster-management** `helmfile` and see the\nGitLab Agent for Kubernetes roll it out with ease!\n\n\n\n## References\n\n- [Create a New GKE\nCluster](https://docs.gitlab.com/ee/user/infrastructure/clusters/connect/new_gke_cluster.html)\n\n- [Create a New EKS\nCluster](https://docs.gitlab.com/ee/user/infrastructure/clusters/connect/new_eks_cluster.html)\n\n- [Cluster Management\nProject](https://docs.gitlab.com/ee/user/clusters/management_project.html)\n\n\n\n## Related posts\n\n- [The ultimate guide to GitOps with\nGitLab](https://about.gitlab.com/blog/the-ultimate-guide-to-gitops-with-gitlab/)\n\n- [GitOps with GitLab: Infrastructure provisioning with GitLab and\nTerraform](https://about.gitlab.com/blog/gitops-with-gitlab-infrastructure-provisioning/)\n\n- [GitOps with GitLab: Connect with a Kubernetes\ncluster](https://about.gitlab.com/blog/gitops-with-gitlab-connecting-the-cluster/)\n",[9,1241,549,787,788,789],{"slug":3955,"featured":6,"template":700},"simple-kubernetes-management-with-gitlab","content:en-us:blog:simple-kubernetes-management-with-gitlab.yml","Simple Kubernetes Management With Gitlab","en-us/blog/simple-kubernetes-management-with-gitlab.yml","en-us/blog/simple-kubernetes-management-with-gitlab",{"_path":3961,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3962,"content":3967,"config":3972,"_id":3974,"_type":13,"title":3975,"_source":15,"_file":3976,"_stem":3977,"_extension":18},"/en-us/blog/simplify-your-cloud-account-management-for-kubernetes-access",{"title":3963,"description":3964,"ogTitle":3963,"ogDescription":3964,"noIndex":6,"ogImage":1875,"ogUrl":3965,"ogSiteName":685,"ogType":686,"canonicalUrls":3965,"schema":3966},"Simplify your cloud account management for Kubernetes access","In this tutorial, learn how to use the GitLab agent for Kubernetes and its user impersonation features for secure cluster access.\n\n","https://about.gitlab.com/blog/simplify-your-cloud-account-management-for-kubernetes-access","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Simplify your cloud account management for Kubernetes access\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Viktor Nagy\"}],\n        \"datePublished\": \"2024-03-19\",\n      }",{"title":3963,"description":3964,"authors":3968,"heroImage":1875,"date":3969,"body":3970,"category":830,"tags":3971},[2244],"2024-03-19","We hear you: Managing cloud accounts is risky, tedious, and time-consuming,\nbut also a must-have in many situations. You might run your Kubernetes\nclusters with one of the hyperclouds, and your engineers need to access at\nleast the non-production cluster to troubleshoot issues quickly and\nefficiently. Sometimes, you also need to give special, temporary access to\nengineers on a production cluster.\n\n\nYou have also told us that access requests might not come very often, but\nwhen they do, they are urgent, and given the high security requirements\naround the process, they can take close to a week to fulfill. \n\n\nBy giving access to your cloud infrastructure, you automatically expose\nyourself to risks. As a result, it's a best practice to restrict access only\nto the resources the given user must have access to. However, cloud identity\nand access management (IAM) is complex by nature. \n\n\nIf you are using Kubernetes and you need to give access specifically to your\nclusters only, GitLab can help. Your user will be able to identify with your\ncluster, so you can configure the Kubernetes role-based access controls\n(RBAC) to restrict their access within the cluster. With GitLab, and\nspecifically the GitLab agent for Kubernetes, you can start at the last step\nand focus only on the RBAC aspect.\n\n\n## What is the GitLab agent for Kubernetes?\n\n\nThe GitLab agent for Kubernetes is a set of GitLab components that allows a\npermanent, bi-directional streaming channel between your GitLab instance and\nyour Kubernetes cluster (one agent per cluster). Once the agent connection\nis configured, you can share it across projects and groups within your\nGitLab instance, allowing a single agent to serve all the access needs of a\ncluster.\n\n\nCurrently, the agent has several features to simplify your Kubernetes\nmanagement tasks:\n\n\n* [Integrates with GitLab\nCI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html)\nfor push-based deployments or regular cluster management jobs. The\nintegration exposes a Kubernetes context per available agent in the Runner\nenvironment, and any tool that can take a context as an input (e.g. kubectl\nor helm CLI) can reach your cluster from the CI/CD jobs.\n\n* Integrates with the GitLab GUI, specifically the environment pages. Users\ncan configure [an environment to show the Kubernetes\nresources](https://docs.gitlab.com/ee/ci/environments/kubernetes_dashboard.html)\navailable in a specific namespace, and even set up a Flux resource to track\nthe reconciliation of your applications.\n\n* Enables users to use the GitLab-managed channel to [connect to the cluster\nfrom their local\nlaptop](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html#access-a-cluster-with-the-kubernetes-api),\nwithout giving them cloud-specific Kubernetes access tokens.\n\n* Supports [Flux GitRepository\nreconciliations](https://docs.gitlab.com/ee/user/clusters/agent/gitops.html#immediate-git-repository-reconciliation)\nby triggering a reconciliation automatically on new commits in repositories\nthe agent can access.\n\n* [Runs operational container\nscans](https://docs.gitlab.com/ee/user/clusters/agent/vulnerabilities.html)\nand shows the reports in the GitLab UI.\n\n* Enables you to enrich the [remote\ndevelopment](https://docs.gitlab.com/ee/user/project/remote_development/)\noffering with [workspaces](https://docs.gitlab.com/ee/user/workspace/).\n\n\n> Try simplifying your cloud account management for Kubernetes access today\nwith [a free trial of GitLab Ultimate](https://gitlab.com/-/trials/new).\n\n\n## The agent and access management\n\n\nThe GitLab agent for Kubernetes, which is available for GitLab Ultimate and\nPremium, impersonates various GitLab-specific users when it acts on behalf\nof GitLab in the cluster.\n\n\n* For the GitLab CI/CD integration, the agent impersonates the CI job as the\nuser, and enriches the user with group specific metadata that describe the\nproject and the group.\n\n\n* For the environment and local connections, the agent impersonates the\nGitLab user using the connection, and similarly to the CI/CD integration,\nthe impersonated Kubernetes user is enriched with group specific metadata,\nlike roles in configured groups.\n\n\nAs this article is about using the agent instead of cloud accounts for\ncluster access, let’s focus on the environment and local connections setup.\n\n\n## An example setup\n\n\nTo offer a realistic setup, let’s assume that in our GitLab instance we have\nthe following groups and projects:\n\n\n* `/app-dev-group/team-a/service-1`\n\n* `/app-dev-group/team-a/service-2`\n\n* `/app-dev-group/team-b/service-3`\n\n* `/platform-group/clusters-project`\n\n\nIn the above setup, the agents are registered against the `clusters-project`\nproject and, in addition to other code, the project contains the agent\nconfiguration files:\n\n\n* `.gitlab/agents/dev-cluster/config.yaml`\n\n* `.gitlab/agents/prod-cluster/config.yaml`\n\n\nThe `dev-cluster` and `prod-cluster` directory names are actually the agent\nnames as well, and registered agents and related events can be seen within\nthe projects “Operations/Kubernetes clusters” menu item. The agent offers\nsome minimal features by default, without a configuration file. To benefit\nfrom the user access features and to share the agent connection across\nprojects and groups, a configuration file is required.\n\n\nLet’s assume that we want to configure the agents in the following way:\n\n\n* For the development cluster connection:\n\n    * Everyone with at least developer role in team-a should be able to read-write their team specific namespace `team-a` only.\n    * Everyone with group owner role in team-a should have namespace admin rights on the `team-a` namespace only.\n    * Members of `team-b` should not be able to access the cluster.\n\n* For the production cluster connection:\n\n    * Everyone with at least developer role in team-a should be able to read-only their team specific namespace `team-a` only.\n    * Members of `team-b` should not be able to access the cluster.\n\nFor the development cluster, the above setup requires an agent configuration\nfile in `.gitlab/agents/dev-cluster/config.yaml` as follows:\n\n\n```yaml\n\nuser_access:\n  access_as:\n    user: {}\n  groups:\n    - id: app-dev-group/team-a # group_id=1\n    - id: app-dev-group/team-b # group_id=2\n```\n\n\nIn this code snippet we added the group ID of the specific groups in a\ncomment. We will need these IDs in the following Kubernetes RBAC\ndefinitions:\n\n\n```yaml\n\napiVersion: rbac.authorization.k8s.io/v1\n\nkind: RoleBinding\n\nmetadata:\n  name: team-a-dev-can-edit\n  namespace: team-a\nroleRef:\n  name: edit\n  kind: ClusterRole\n  apiGroup: rbac.authorization.k8s.io\nsubjects:\n  - name: gitlab:group_role:1:developer\n    kind: Group\n```\n\n\nand...\n\n\n```yaml\n\napiVersion: rbac.authorization.k8s.io/v1\n\nkind: RoleBinding\n\nmetadata:\n  name: team-a-owner-can-admin\n  namespace: team-a\nroleRef:\n  name: admin\n  kind: ClusterRole\n  apiGroup: rbac.authorization.k8s.io\nsubjects:\n  - name: gitlab:group_role:1:owner\n    kind: Group\n```    \n\n\nThe above two code snippets can be applied to the cluster with the GitLab\nFlux integration or manually via `kubectl`. They describe role bindings for\nthe `team-a` group members. It’s important to note that only the groups and\nprojects from the agent configuration file can be targeted as RBAC groups.\nTherefore, the following RBAC will not work as the impersonated user\nresources don’t know about the referenced projects:\n\n\n```yaml\n\napiVersion: rbac.authorization.k8s.io/v1\n\nkind: RoleBinding\n\nmetadata:\n  name: team-a-dev-can-edit\n  namespace: team-a\nroleRef:\n  name: edit\n  kind: ClusterRole\n  apiGroup: rbac.authorization.k8s.io\nsubjects:\n  - name: gitlab:project_role:3:developer # app-dev-group/team-a/service-1 project ID is 3\n    kind: Group\n```\n\n\nFor the production cluster we need the same agent configuration under\n`.gitlab/agents/prod-cluster/config.yaml` and the following RBAC\ndefinitions:\n\n\n```yaml\n\napiVersion: rbac.authorization.k8s.io/v1\n\nkind: RoleBinding\n\nmetadata:\n  name: team-a-dev-can-read\n  namespace: team-a\nroleRef:\n  name: view\n  kind: ClusterRole\n  apiGroup: rbac.authorization.k8s.io\nsubjects:\n  - name: gitlab:group_role:1:developer\n    kind: Group\n```\n\n\nThese configurations allow project owners to set up the environment pages so\nmembers of `team-a` will be able to see the status of their cluster\nworkloads in real-time and they should be able to access the cluster from\ntheir local computers using their favorite Kubernetes tools.\n\n\n## Explaining the magic\n\n\nIn the previous section, you learned how to set up role bindings for group\nmembers with specific roles. In this section, let's dive into the\nimpersonated user and their attributes.\n\n\nWhile Kubernetes does not have a User or Group resource, its authentication\nand authorization scheme pretends to have it. Users have a username, can\nbelong to groups, and can have other extra attributes.\n\n\nThe impersonated GitLab user carries the `gitlab:username:\u003Cusername>` in the\ncluster. For example, if our imaginary user Béla has the GitLab username\n`bela`, then in the cluster the impersonated user will be called\n`gitlab:username:bela`. This allows targeting of a specific user in the\ncluster.\n\n\nEvery impersonated user belongs to the `gitlab:user` group. Moreover, for\nevery project and group listed in the agent configuration, we check the\ncurrent user’s role and add it as a group. This is more easily understood\nthrough an example, so let’s modify a little bit the agent configuration we\nused above.\n\n\n```yaml\n\nuser_access:\n  access_as:\n    user: {}\n  projects:\n    - id: platform-group/clusters-project # project_id=1\n  groups:\n    - id: app-dev-group/team-a # group_id=1\n    - id: app-dev-group/team-b # group_id=2\n```\n\n\nFor the sake of example, let’s assume the contrived setup that our user Béla\nis a maintainer in the `platform-group/clusters-project` project, is a\ndeveloper in `app-dev-group/team-a` group, and an owner of the\n`app-dev-group/team-a/service-1` project. In this case, the impersonated\nKubernetes user `gitlab:username:bela` will belong to the following groups:\n\n\n* `gitlab:user`\n\n* `gitlab:project_role:1:developer`\n\n* `gitlab:project_role:1:maintainer`\n\n* `gitlab:group_role:1:developer`\n\n\nWhat happens is that we check Béla’s role in every project and group listed\nin the agent configuration, and set up all the roles that Béla has there. As\nBéla is a maintainer in `platform-group/clusters-project` (project ID 1), we\nadd him to both the `gitlab:project_role:1:developer` and\n`gitlab:project_role:1:maintainer` groups. Note as well, that we did not add\nany groups for the `app-dev-group/team-a/service-1` project, only its parent\ngroup that appears in the agent configuration.\n\n\n## Simplifying cluster management\n\n\nSetting up the agent and configuring the cluster as presented above is\neverything you need to model the presented access requirements in the\ncluster. You don’t have to manage cloud accounts or add in-cluster account\nmanagement tools like Dex. The agent for Kubernetes and its user\nimpersonation features can simplify your infrastructure management work.\n\n\nWhen new people join your company, once they become members of the `team-a`\nthey immediately get access to the clusters as configured above. Similarly,\nas someone leaves your company, you just have to remove them from the group\nand their access will be disabled. As we mentioned, the agent supports local\naccess to the clusters, too. As that local access runs through the\nGitLab-side agent component, it will be disabled as well when users are\nremoved from the `team-a` group.\n\n\nSetting up the agent takes around two-to-five minutes per cluster. Setting\nup the required RBAC might take another five minutes. In 10 minutes, users\ncan get controlled access to a cluster, saving days of work and decreasing\nthe risks associated with cloud accounts.\n\n\n## Get started today\n\n\nIf you want to try this approach and allow access to your colleagues to some\nof your clusters without managing cloud accounts, the following\ndocumentation pages should help you to get started:\n\n\n- On self-managed GitLab instances, you might need to [configure the\nGitLab-side component (called\nKAS)](https://docs.gitlab.com/ee/administration/clusters/kas.html) of the\nagent for Kubernetes first.\n\n\n- You can learn more about [all the Kubernetes management features\nhere](https://docs.gitlab.com/ee/user/clusters/agent/), or you can\nimmediately dive in by [installing an\nagent](https://docs.gitlab.com/ee/user/clusters/agent/install/), and\n[granting users access to\nKubernetes](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html).\n\n\n- You’ll likely want to [configure a Kubernetes\ndashboard](https://docs.gitlab.com/ee/ci/environments/kubernetes_dashboard.html)\nfor your deployed application.\n\n\n> Try simplifying your cloud account management for Kubernetes access today\nwith [a free trial of GitLab Ultimate](https://gitlab.com/-/trials/new).\n",[1898,830,1241,9],{"slug":3973,"featured":90,"template":700},"simplify-your-cloud-account-management-for-kubernetes-access","content:en-us:blog:simplify-your-cloud-account-management-for-kubernetes-access.yml","Simplify Your Cloud Account Management For Kubernetes Access","en-us/blog/simplify-your-cloud-account-management-for-kubernetes-access.yml","en-us/blog/simplify-your-cloud-account-management-for-kubernetes-access",{"_path":3979,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3980,"content":3983,"config":3988,"_id":3990,"_type":13,"title":3991,"_source":15,"_file":3992,"_stem":3993,"_extension":18},"/en-us/blog/software-supply-chain-security-guide-why-organizations-struggle",{"noIndex":6,"title":3981,"description":3982},"Software supply chain security guide: Why organizations struggle","Part 1 of this new series explores fundamental challenges, practical solutions, and emerging trends, including AI, that every development team needs to understand.",{"title":3981,"description":3982,"authors":3984,"heroImage":1816,"date":3985,"body":3986,"category":830,"tags":3987},[1096],"2025-07-24","Ask most development teams about supply chain security, and you'll get answers focused on vulnerability scanning or dependency management. While these are components of supply chain security, they represent a dangerously narrow view of a much broader challenge.\n\n**Supply chain security isn't just about scanning dependencies.** It encompasses the entire journey from code creation to production deployment, including:\n\n* **Source security:** protect code repositories, managing contributor access, ensuring code integrity  \n* **Build security:** secure build environments, preventing tampering during compilation and packaging  \n* **Artifact security:** ensure the integrity of containers, packages, and deployment artifacts  \n* **Deployment security:** secure the delivery mechanisms and runtime environments  \n* **Tool security:** harden the development tools and platforms themselves\n\nThe \"chain\" in supply chain security refers to this interconnected series of steps. A weakness anywhere in the chain can compromise the entire software delivery process.\n\nThe [2020 SolarWinds attack](https://www.cisa.gov/news-events/news/joint-statement-federal-bureau-investigation-fbi-cybersecurity-and-infrastructure-security) illustrates this perfectly. In what became one of the largest supply chain attacks in history, state-sponsored attackers compromised the build pipeline of SolarWinds' Orion network management software. Rather than exploiting a vulnerable dependency or hacking the final application, they injected malicious code during the compilation process itself.\n\nThe result was devastating: More than 18,000 organizations, including multiple U.S. government agencies, unknowingly installed backdoored software through normal software updates. The source code was clean, the final application appeared legitimate, but the build process had been weaponized. This attack remained undetected for months, demonstrating how supply chain vulnerabilities can bypass traditional security measures.\n\n### Common misconceptions that leave organizations vulnerable\n\nDespite growing awareness of supply chain threats, many organizations remain exposed because they operate under fundamental misunderstandings about what software supply chain security actually entails. These misconceptions create dangerous blind spots:\n\n* Thinking software supply chain security equals dependency scanning  \n* Focusing only on open source components while ignoring proprietary code risks  \n* Believing that code signing alone provides sufficient protection  \n* Assuming that secure coding practices eliminate supply chain risks  \n* Treating it as a security team problem rather than a development workflow challenge\n\n![Software supply chain security dependency chart](https://res.cloudinary.com/about-gitlab-com/image/upload/v1753200077/kqndvlxyvncshdiq0xea.png)\n\n## How AI is changing the game\n\nJust as organizations are grappling with traditional software supply chain security challenges, artificial intelligence (AI)  is introducing entirely new attack vectors and amplifying existing ones in unprecedented ways.\n\n### AI-powered attacks: More sophisticated, more scalable\n\nAttackers are using AI to automate vulnerability discovery, generate convincing social engineering attacks targeting developers, and systematically analyze public codebases for weaknesses. What once required manual effort can now be done at scale — with precision.\n\n### The AI development supply chain introduces new risks\n\nAI is reshaping the entire development lifecycle, but it's also introducing significant security blind spots:\n\n* **Model supply chain attacks:** Pre-trained models from sources like Hugging Face or GitHub may contain backdoors or poisoned training data.  \n* **Insecure AI-generated code:** Developers using AI coding assistants may unknowingly introduce vulnerable patterns or unsafe dependencies.  \n* **Compromised AI toolchains:** The infrastructure used to train, deploy, and manage AI models creates a new attack surface.  \n* **Automated reconnaissance:** AI enables attackers to scan entire ecosystems to identify high-impact supply chain targets.  \n* **Shadow AI and unsanctioned tools:** Developers may integrate external AI tools that haven't been vetted.\n\nThe result? AI doesn't just introduce new vulnerabilities, it amplifies the scale and impact of existing ones. Organizations can no longer rely on incremental improvements. The threat landscape is evolving faster than current security practices can adapt.\n\n![AI amplification effect](https://res.cloudinary.com/about-gitlab-com/image/upload/v1753200139/xuxezxld6ztlvjocgjlx.png)\n\n## Why most organizations still struggle\n\nEven organizations that understand supply chain security often fail to act effectively. The statistics reveal a troubling pattern of awareness without corresponding behavior change.\n\nWhen [Colonial Pipeline paid hackers $4.4 million](https://www.cnn.com/2021/05/19/politics/colonial-pipeline-ransom/index.html) in 2021 to restore operations, or when 18,000 organizations fell victim to the SolarWinds attack, the message was clear: Supply chain vulnerabilities can bring down critical infrastructure and compromise sensitive data at unprecedented scale.\n\nYet, despite this awareness, most organizations continue with business as usual. The real question isn't whether organizations care about supply chain security — it's why caring alone isn't translating into effective protection.\n\nThe answer lies in four critical barriers that prevent effective action:\n\n**1. The false economy mindset**\n\nOrganizations sometimes focus on the cost instead of \"what's the most effective approach?\" This cost-first thinking creates expensive downstream problems.\n\n**2. Skills shortage reality**\n\nWith [organizations averaging 4 security professionals per 100 developers](https://codific.com/bsimm-building-security-in-maturity-model-a-complete-guide/), according to BSIMM research, and [90% of organizations reporting critical cybersecurity skills gaps](https://www.isc2.org/Insights/2024/09/Employers-Must-Act-Cybersecurity-Workforce-Growth-Stalls-as-Skills-Gaps-Widen), according to ISC2, traditional approaches are mathematically impossible to scale.\n\n**3. Misaligned organizational incentives**\n\nDeveloper OKRs focus on feature velocity while security teams measure different outcomes. When C-suite priorities emphasize speed-to-market over security posture, friction becomes inevitable.\n\n**4. Tool complexity overload**\n\nThe [average enterprise uses 45 cybersecurity tools](https://www.gartner.com/en/newsroom/press-releases/2025-03-03-gartner-identifiesthe-top-cybersecurity-trends-for-2025), with [40% of security alerts being false positives](https://www.ponemon.org/news-updates/blog/security/new-ponemon-study-on-malware-detection-prevention-released.html) and must [coordinate across 19 tools on average for each incident](https://newsroom.ibm.com/2020-06-30-IBM-Study-Security-Response-Planning-on-the-Rise-But-Containing-Attacks-Remains-an-Issue).\n\nThese barriers create a vicious cycle: Organizations recognize the threat, invest in security solutions, but implement them in ways that don't drive the desired outcomes.\n\n## The true price of supply chain insecurity\n\nSupply chain attacks create risk and expenses that extend far beyond initial remediation. Understanding these hidden multipliers helps explain why prevention is not just preferable – it's essential for business continuity.\n\n**Time becomes the enemy**\n\n* Average time to identify and contain a supply chain breach: [277 days](https://keepnetlabs.com/blog/171-cyber-security-statistics-2024-s-updated-trends-and-data)  \n* Customer trust rebuilding period: [2-3+ years](https://www.bcg.com/publications/2024/rebuilding-corporate-trust)   \n* Engineering hours diverted from product development to security remediation\n\n**Reputation damage compounds** \n\nWhen attackers compromise your supply chain, they don't just steal data – they undermine the foundation of customer trust. [Customer churn rates typically increase 33% post-breach](https://www.metacompliance.com/blog/data-breaches/5-damaging-consequences-of-a-data-breach), while partner relationships require costly re-certification processes. Competitive positioning suffers as prospects choose alternatives perceived as \"safer.\"\n\n**Regulatory reality bites** \n\nThe regulatory landscape has fundamentally shifted. [GDPR fines now average over $50 million for significant data breache](https://www.skillcast.com/blog/20-biggest-gdpr-fines)s. The EU's new [Cyber Resilience Act](https://about.gitlab.com/blog/gitlab-supports-banks-in-navigating-regulatory-challenges/#european-cyber-resilience-act-\\(cra\\)) mandates supply chain transparency. U.S. federal contractors must provide software bills of materials ([SBOMs](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/)) for all software purchases — a requirement that's rapidly spreading to private sector procurement.\n\n**Operational disruption multiplies** \n\nBeyond the direct costs, supply chain attacks create operational chaos such as platform downtime during attack remediation, emergency security audits across entire technology stacks, and legal costs from customer lawsuits and regulatory investigations.\n\n## What's wrong with current approaches\n\nMost organizations confuse security activity with security impact. They deploy scanners, generate lengthy reports, and chase teams to address through manual follow-ups. But these efforts often backfire — creating more problems than they solve.\n\n### Massive scanning vs. effective protection\n\nEnterprises generate over [10,000 security alerts each month, with the most active generating roughly 150,000 events per day.](https://www.securityweek.com/enterprises-generate-10000-security-events-day-average-report/) [But 63%](https://panther.com/blog/identifying-and-mitigating-false-positive-alerts) of these are false positives or low-priority noise. Security teams become overwhelmed and turn into bottlenecks instead of enablers.\n\n### The collaboration breakdown\n\nThe most secure organizations don't have the most tools; they have the strongest DevSecOps collaboration. But most current setups make this harder by splitting workflows across incompatible tools, failing to show developers security results in their environment, and offering no shared visibility into risk and business impact.\n\n## The path forward\n\nUnderstanding these challenges is the first step toward building effective supply chain security. The organizations that succeed don't just add more security tools, they fundamentally rethink how security integrates with development workflows. They also review end-to-end software delivery workflows to simplify processes, reduce tools and improve collaboration.\n\nAt GitLab, we've seen how integrated DevSecOps platforms can address these challenges by bringing security directly into the development workflow. In our next article in this series, we'll explore how leading organizations are transforming their approach to supply chain security through developer-native solutions, AI-powered automation, and platforms that make security a natural part of building great software.\n> Learn more about [GitLab's software supply chain security capabilities](https://about.gitlab.com/solutions/supply-chain/).",[830,694,9],{"featured":90,"template":700,"slug":3989},"software-supply-chain-security-guide-why-organizations-struggle","content:en-us:blog:software-supply-chain-security-guide-why-organizations-struggle.yml","Software Supply Chain Security Guide Why Organizations Struggle","en-us/blog/software-supply-chain-security-guide-why-organizations-struggle.yml","en-us/blog/software-supply-chain-security-guide-why-organizations-struggle",{"_path":3995,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":3996,"content":4001,"config":4005,"_id":4007,"_type":13,"title":4008,"_source":15,"_file":4009,"_stem":4010,"_extension":18},"/en-us/blog/speed-up-code-reviews-let-ai-handle-the-feedback-implementation",{"title":3997,"description":3998,"ogTitle":3997,"ogDescription":3998,"noIndex":6,"ogImage":1781,"ogUrl":3999,"ogSiteName":685,"ogType":686,"canonicalUrls":3999,"schema":4000},"Speed up code reviews: Let AI handle the feedback implementation","Discover how GitLab Duo with Amazon Q automates the implementation of code review feedback through AI, transforming a time-consuming manual process into a streamlined workflow.","https://about.gitlab.com/blog/speed-up-code-reviews-let-ai-handle-the-feedback-implementation","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Speed up code reviews: Let AI handle the feedback implementation\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2025-06-10\",\n      }",{"title":3997,"description":3998,"authors":4002,"heroImage":1781,"date":872,"body":4003,"category":718,"tags":4004},[715],"You know that feeling when you've just submitted a merge request and the code review comments start rolling in? One reviewer wants the labels updated, another asks for side-by-side layouts, someone else requests bold formatting, and don't forget about that button color change. Before you know it, you're spending hours implementing feedback that, while important, takes you away from building new features. It's a time-consuming process that every developer faces, yet it feels like there should be a better way.\n\nWhat if you could have an AI assistant that understands code review feedback and automatically implements the changes for you? That's exactly what [GitLab Duo with Amazon Q](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/) brings to your development workflow. This seamless integration combines GitLab's comprehensive DevSecOps platform with Amazon Q's advanced AI capabilities, creating an intelligent assistant that can read reviewer comments and converts them directly into code changes. Instead of manually addressing each piece of feedback, you can let AI handle the implementation while you focus on the bigger picture.\n\n## How GitLab Duo with Amazon Q works\n\nWhen you're viewing a merge request with reviewer comments, you'll see feedback scattered throughout your code. Let's take the examples from earlier in this article: maybe you've received a request to update a form label here, a suggestion to display fields side-by-side there, or a note about making certain text bold. Each comment represents a task that normally you'd need to handle manually.\n\n![feedback on an MR](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749673634/Blog/Content%20Images/1-show-comment.png)\n\nWith GitLab Duo with Amazon Q, you can simply enter the `/q dev` quick action in a comment. This prompts Amazon Q to analyze all the feedback and start modifying your code automatically. The AI agent understands the context of each comment and implements the requested changes directly in your codebase.\n\n![/q dev function prompting Amazon Q to analyze feedback](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749673634/Blog/Content%20Images/2-invoke-q-dev.png)\n\nOnce Amazon Q processes the feedback, you can view all the updates in the \"Changes\" tab of your merge request. Every modification is clearly visible, so you can verify that the AI agent correctly interpreted and implemented each piece of feedback. You can then run your updated application to confirm that all the changes work as expected — that form label is updated, the fields are displayed side-by-side, the text is bold, and yes, that button is now blue.\n\nWatch the code review feedback process in action:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/31E9X9BrK5s?si=ThFywR34V3Bfj1Z-\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nProcessing code review feedback is a necessary but time-intensive part of software development.  GitLab Duo with Amazon Q evolves this manual process into an automated workflow, dramatically reducing the time between receiving feedback and implementing changes. By letting AI handle these routine modifications, you're free to focus on what really matters — building innovative features and solving complex problems.\n\nWith GitLab Duo with Amazon Q, you can:\n- Eliminate hours of manual feedback implementation\n- Accelerate your code review cycles\n- Maintain consistency in how feedback is addressed\n- Reduce context switching between reviewing comments and writing code\n- Ship features faster with streamlined deployment times\n\n> #### To learn more about GitLab Duo with Amazon Q visit us at an upcoming [AWS Summit in a city near you](https://about.gitlab.com/events/aws-summits/) or [reach out to your GitLab representative](https://about.gitlab.com/partners/technology-partners/aws/#form).\n\n## GitLab Duo with Amazon Q resources\n\n- [GitLab Duo with Amazon Q: Agentic AI optimized for AWS generally available](https://about.gitlab.com/blog/gitlab-duo-with-amazon-q-agentic-ai-optimized-for-aws/)\n- [GitLab and AWS partner page](https://about.gitlab.com/partners/technology-partners/aws/)\n- [GitLab Duo with Amazon Q documentation](https://docs.gitlab.com/user/duo_amazon_q/)\n- [What is agentic AI?](https://about.gitlab.com/topics/agentic-ai/)\n- [Agentic AI guides and resources](https://about.gitlab.com/blog/agentic-ai-guides-and-resources/)",[694,696,495,810,809,9],{"slug":4006,"featured":90,"template":700},"speed-up-code-reviews-let-ai-handle-the-feedback-implementation","content:en-us:blog:speed-up-code-reviews-let-ai-handle-the-feedback-implementation.yml","Speed Up Code Reviews Let Ai Handle The Feedback Implementation","en-us/blog/speed-up-code-reviews-let-ai-handle-the-feedback-implementation.yml","en-us/blog/speed-up-code-reviews-let-ai-handle-the-feedback-implementation",{"_path":4012,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4013,"content":4019,"config":4025,"_id":4027,"_type":13,"title":4028,"_source":15,"_file":4029,"_stem":4030,"_extension":18},"/en-us/blog/speed-up-your-monorepo-workflow-in-git",{"title":4014,"description":4015,"ogTitle":4014,"ogDescription":4015,"noIndex":6,"ogImage":4016,"ogUrl":4017,"ogSiteName":685,"ogType":686,"canonicalUrls":4017,"schema":4018},"Speed up your monorepo workflow in Git","Tap into the features that can reap huge savings in the long run for any developer team.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749665560/Blog/Hero%20Images/speedmonorepo.jpg","https://about.gitlab.com/blog/speed-up-your-monorepo-workflow-in-git","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Speed up your monorepo workflow in Git\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"John Cai\"}],\n        \"datePublished\": \"2022-09-06\",\n      }",{"title":4014,"description":4015,"authors":4020,"heroImage":4016,"date":4022,"body":4023,"category":785,"tags":4024},[4021],"John Cai","2022-09-06","Monorepos have grown in popularity in recent years. For many of us, they are\na\n\npart of our daily Git workflows. The trouble is working with them can be\nslow. Speeding up\n\na developer's workflow can reap huge savings in the long run for any team.\n\n\nFirst, a word about monorepos. What does it mean for a repository to be a\n\nmonorepo anyway? Well, it depends who you ask and the definition has become\n\nmore flexible over time, but here are a few.\n\n\n## Characteristics of monorepos\n\n\nMonorepos have the following characteristics.\n\n\n### Multiple sub-projects\n\n\nThe typical definition of \"monorepo\" is a repository that contains multiple\nsub-projects. For instance, let's imagine a repository with a web-facing\nfront end,\n\na backend, an iOS app directory, and an android app directory:\n\n\n```\n\nawesome-app/\n\n|\n\n|--backend/\n\n|\n\n|--web-frontend/\n\n|\n\n|--app-ios/\n\n|\n\n|--app-android/\n\n\n```\n\n\n`awesome-app` is a single repository:\n\n\n```\n\ngit clone https://my-favorite-git-hosting-service.com/awesome-app.git\n\n```\n\n\nThe [Chromium](https://github.com/chromium/chromium) repository is a good\n\nexample of this.\n\n\n### Large files\n\n\nRepositories can also grow to be very large if large files are checked in.\nIn\n\nsome cases, binaries or other large assets such as images are checked into\nthe\n\nrepository to have their history tracked. Other times, large files are\ninadvertently \n\nintroduced into the repository. The way Git history works, even if these\nfiles are\n\nimmediately removed, the single version that was checked in remains.\n\n\n### Old projects with deep histories\n\n\nWhile Git is very good at compressing text files, when a Git repository has\na deep history,\n\nthe need to keep all versions of a file around can cause the size of the\nrepository to be huge.\n\n\nThe [Linux](https://github.com/torvalds/linux) repository is a good example\nof this.\n\n\nFor instance, the Linux project's first Git commit is from [April\n2005](https://github.com/torvalds/linux/commit/1da177e4c3f41524e886b7f1b8a0c1fc7321cac2).\n\n\nAnd a `git rev-list --all --count` gives us 1,120,826 commits! That's a lot\nof\n\nhistory! Getting into Git internals a little bit, Git keeps a commit object,\nand a\n\ntree object for each commit, as well as a copy of the files at that snapshot\n\nin history. This means a deep Git history means a lot of Git data.\n\n\n## Speeding up your Git workflow\n\n\nHere are some features to help speed up your Git workflow.\n\n\n### Sparse checkout\n\n\n[git sparse checkout](https://git-scm.com/docs/git-sparse-checkout) reduces\nthe\n\nnumber of files you check out to a subset of the repository. (NOTE: This\nfeature\n\nin Git is still marked experimental.) This is especially useful in the case\nof\n\n[many sub-projects in a repository](#multiple-sub-projects).\n\n\nTaking our [example](#multiple-sub-projects) of a monorepo with multiple\n\nsub-projects, let's say that as a front-end web developer I only need to\nmake\n\nchanges to `web-frontend/`.\n\n\n```sh\n\n> git clone --no-checkout\nhttps://my-favorite-git-hosting-service.com/awesome-app.git\n\n> cd awesome-app\n\n> git sparse-checkout set web-frontend\n\n> git checkout\n\nYour branch is up to date with 'origin/master'.\n\n> ls\n\n> web-frontend README.md\n\n```\n\n\nOr, if you've already checked out a worktree, sparse checkout can be used to\nremove\n\nfiles from the worktree.\n\n\n\n```sh\n\n> git clone https://my-favorite-git-hosting-service.com/awesome-app.git\n\n> cd awesome-app\n\n> ls\n\n> backend web-frontend app-ios app-android README.md\n\n> git sparse-checkout set web-frontend\n\nUpdating files: 100% (103452/103452), done.\n\n> ls\n\n> web-frontend README.md\n\n```\n\n\nSparse checkout will only include the directories indicated, plus all files\n\ndirectly under the root repository directory.\n\n\nThis way, we only checkout the directories that we need, saving both space\nlocally\n\nand time since each time `git pull` is done, only files that are checked out\nwill\n\nneed to be updated.\n\n\nMore information can be found in the\n[docs](https://git-scm.com/docs/git-sparse-checkout)\n\nfor sparse checkout.\n\n\n### Partial clone\n\n\n[git partial\nclone](https://docs.gitlab.com/ee/topics/git/partial_clone.html#:~:text=Partial%20clone%20is%20a%20performance,0%20or%20later%20is%20required)\nhas a similar goal to sparse checkout in reducing the number\n\nof files in your local Git repository. It provides the option to filter out\n\ncertain types of files when cloning.\n\n\nPartial clone is used by passing the `--filter` option to `git-clone`.\n\n\n```sh\n\ngit clone --filter=blob:limit=10m\n\n```\n\n\nThis will exclude any files over 10 megabytes from being copied to the local\n\nrepository. A full list of supported filters are included in the\n\n[docs for\ngit-rev-list](https://git-scm.com/docs/git-rev-list#Documentation/git-rev-list.txt\n",[742,956,9],{"slug":4026,"featured":6,"template":700},"speed-up-your-monorepo-workflow-in-git","content:en-us:blog:speed-up-your-monorepo-workflow-in-git.yml","Speed Up Your Monorepo Workflow In Git","en-us/blog/speed-up-your-monorepo-workflow-in-git.yml","en-us/blog/speed-up-your-monorepo-workflow-in-git",{"_path":4032,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4033,"content":4038,"config":4042,"_id":4044,"_type":13,"title":4045,"_source":15,"_file":4046,"_stem":4047,"_extension":18},"/en-us/blog/streamline-devsecops-engineering-workflows-with-gitlab-duo",{"title":4034,"description":4035,"ogTitle":4034,"ogDescription":4035,"noIndex":6,"ogImage":863,"ogUrl":4036,"ogSiteName":685,"ogType":686,"canonicalUrls":4036,"schema":4037},"Streamline DevSecOps engineering workflows with GitLab Duo","Learn all the ways GitLab Duo's AI capabilities can improve the efficiency of development workflows. Includes in-depth tutorials and demos.","https://about.gitlab.com/blog/streamline-devsecops-engineering-workflows-with-gitlab-duo","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Streamline DevSecOps engineering workflows with GitLab Duo\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2024-12-05\",\n      }",{"title":4034,"description":4035,"authors":4039,"heroImage":863,"date":2970,"body":4040,"category":718,"tags":4041},[1604],"It's 9 a.m. somewhere, and a DevOps engineer is starting their day. They check their [GitLab todo list](https://docs.gitlab.com/ee/user/todos.html) to see any mentions or tasks assigned to them, collaborating with other stakeholders in their organization. These tasks can include:\n\n- managing infrastructure\n- maintaining the configuration of resources\n- maintaining CI/CD pipelines\n- automating processes for efficiency\n- maintaining monitoring and alerting systems\n- ensuring applications are securely built and deployed\n- modernizing applications with containerization\n\nTo carry out these tasks, DevOps engineers spend a lot of time reading documentation, writing configuration files, and searching for help in forums, issues boards, and blogs. Time is spent studying and understanding concepts, and how tools and technologies work. When they don't work as expected, a lot more time is spent investigating why. New tools are released regularly to solve niche or existing problems differently, which introduces more things to learn and maintain context for.\n\n[GitLab Duo](https://about.gitlab.com/gitlab-duo/), our AI-powered suite of capabilities, fits into the workflow of DevSecOps engineers, enabling them to reduce time spent solving problems while increasing their efficiency.\n\nLet's explore how GitLab Duo helps streamline workflows.\n\n## Collaboration and communication\n\nDiscussions or requests for code reviews require spending time reading comments from everyone and carefully reviewing the work shared. GitLab Duo capabilities like Discussion Summary, Code Review Summary, and Merge Request Summary increase the effectiveness of collaboration by reducing the time required to get caught up on activities and comments, with more time spent getting the actual work done.\n\n### Merge Request Summary  \n\nWriting a detailed and clear summary of the change a merge request introduces is crucial for every stakeholder to understand what, why, and how a change was made. It's more difficult than it sounds to effectively articulate every change made, especially in a large merge request. [Merge Request Summary](https://docs.gitlab.com/ee/user/project/merge_requests/duo_in_merge_requests.html#generate-a-description-by-summarizing-code-changes) analyzes the change's diff and provides a detailed summary of the changes made.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/4muvSFuWWL4?si=1i2pkyqXZGn2dSbd\" title=\"GitLab Duo Chat is now aware of Merge Requests\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Discussion Summary\n\nImagine getting pulled into an issue with more than 100 comments and a lengthy description, with different perspectives and opinions shared. GitLab Duo [Discussion Summary](https://docs.gitlab.com/ee/user/discussions/index.html#summarize-issue-discussions-with-duo-chat) summarizes all the conversations in the issue and identifies tasks that need to be done, reducing time spent. \n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/IcdxLfTIUgc?si=WXlINow3pLoKHBVM\" title=\"GitLab Duo Dicussion Summary\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n#### Code Review Summary\n\nA merge request has been assigned to a DevOps engineer for review in preparation for deployment, and they have spent time reviewing several parts of the change with multiple comments and suggestions. When [submitting a review](https://docs.gitlab.com/ee/user/project/merge_requests/reviews/index.html#submit-a-review), a text box is presented to summarize the review, which often requires taking a pause and articulating the review. With [Code Review Summary](https://docs.gitlab.com/ee/user/project/merge_requests/reviews/index.html#submit-a-review), they get a concise summary automatically drafted leading to efficiency.\n\n## Manage infrastructure changes\n\nPart of a DevOps engineer's workflow is managing infrastructure changes. Infrastructure as code ([IaC](https://docs.gitlab.com/ee/user/infrastructure/iac/)) revolutionized this process, allowing for documentation, consistency, faster recovery, accountability, and collaboration. A challenge with IaC is understanding the requirements and syntax of the chosen tool and provider where the infrastructure will be created. A lot of time is then spent reviewing documentation and tweaking configuration files until they meet expectations. \n\nWith GitLab Duo [Code Explanation](https://docs.gitlab.com/ee/user/gitlab_duo/index.html#code-explanation) and [Code Suggestions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/index.html), you can prompt GitLab Duo to create configuration files in your tool of choice and learn about the syntax of those tools. With Code Suggestions, you can either leverage [code generation](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/index.html#code-generation), where you prompt GitLab Duo to generate the configuration, or code completion, which provides suggestions as you type while maintaining the context of your existing configurations.\n\nAs of the time this article was published, Terraform is [supported by default](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html#supported-languages) with the right extensions for your IDEs. Other technologies can be supported with [additional language support configuration](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html#add-support-for-more-languages) for the [GitLab Workflow extension](https://docs.gitlab.com/ee/editor_extensions/visual_studio_code/index.html).\n\nWhere a technology is not officially supported, [GitLab Duo Chat](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html) is the powerful AI assistant that can help generate, explain, clarify, and troubleshoot your configuration, while maintaining context from selected text or opened files. Here are two demos where GitLab Duo helped create IaC with Terraform and AWS CloudFormation.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/saa2JJ57UaQ?si=Bu9jyQWwuSUcw8vr\" title=\"Manage your Infrastructure with Terraform and AI using GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n\u003Cbr>\u003C/br>\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/KSLk2twXqiI?si=QDdERjbM0f7X2p23\" title=\"Deploying AWS Lambda function using AWS Cloudformation with help from GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Configuration management\n\nOnce your infrastructure is up, GitLab Duo Chat can also help create configuration files and refactor existing ones. These can be Ansible configurations for infrastructure or cloud-native configurations using Docker, Kubernetes, or Helm resource files. In the videos below, I demonstrate how GitLab Duo helps with Ansible, containerization, and application deployment to Kubernetes.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/t6ZCq_jkBwY?si=awCUdu1wCgOO21XR\" title=\"Configuring your Infrastructure with Ansible & GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n\u003Cbr>\u003C/br>\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/KSLk2twXqiI?si=QDdERjbM0f7X2p23\" title=\"Containerizing your application with GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n\u003Cbr>\u003C/br>\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/uroSxvMFqPU?si=GMNC7f2b7i_cjn6F\" title=\"Deploying your application to Kubernetes with Help from GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n\u003Cbr>\u003C/br>\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/9yGDM00RlUA?si=kE5JZD_OEFcxeR7E\" title=\"Deploying to Kubernetes using Helm with help from GitLab Duo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Test, test, test\n\nWriting tests is an important part of building secure software, but it can be a chore and often becomes an afterthought. You can leverage the power of GitLab Duo to [generate tests for your code](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/index.html#vulnerability-resolution) by highlighting your code and typing the `/tests` in the Chat panel of your IDE.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/zWhwuixUkYU?si=wI93j90PIiUMyGcV\" title=\"GitLab Duo Test Generation\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### CI/CD pipeline troubleshooting\n\nAutomation is an essential part of the DevOps engineer's workflow, and Continuous Integration/Deployment ([CI/CD](https://about.gitlab.com/topics/ci-cd/)) is central to this. You can trigger CI jobs on code push, merge, or on schedule. But, when jobs fail, you spend a lot of time reading through the logs to identify why, and for cryptic errors, it can take more time to figure out. [GitLab Duo Root Cause Analysis](https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd/) analyzes your failed job log and errors, and then recommends possible fixes. This reduces the time spent investigating the errors and finding a fix.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Sa0UBpMqXgs?si=IyR-skz9wJMBSicE\" title=\"GitLab Duo Root Cause Analysis\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Building secure applications\n\nPart of software development includes discovering vulnerabilities, either in the application or its dependencies. Some vulnerabilities are easy to fix, while others require creating a milestone with planning. GitLab Duo [Vulnerability Explanation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/index.html#explaining-a-vulnerability) and [Vulnerability Resolution](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/index.html#vulnerability-resolution) reduce the time spent researching and fixing vulnerabilities. Vulnerability Explanation explains why a vulnerability is happening, its impact, and how to fix it, helping the DevOps engineer to upskill. Vulnerability Resolution takes it further – instead of just suggesting a fix, it creates a merge request with a fix for the vulnerability for you to review. \n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/MMVFvGrmMzw?si=Fxc4SeOkCBKwUk_k\" title=\"GitLab Duo Vulnerability Explanation\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n\u003Cbr>\u003C/br>\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/VJmsw_C125E?si=XT3Qz5SsX-ISfCyq\" title=\"GitLab Duo Vulnerability resolution\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## More work done with less stress\n\nWith GitLab Duo, DevOps engineers can do more work deploying and maintaining secure applications, while acquiring more skills with the detailed responses from GitLab Duo Chat.\n\n> [Sign up for a free trial of GitLab Duo](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/) to get started today!",[720,108,851,495,9,694],{"slug":4043,"featured":90,"template":700},"streamline-devsecops-engineering-workflows-with-gitlab-duo","content:en-us:blog:streamline-devsecops-engineering-workflows-with-gitlab-duo.yml","Streamline Devsecops Engineering Workflows With Gitlab Duo","en-us/blog/streamline-devsecops-engineering-workflows-with-gitlab-duo.yml","en-us/blog/streamline-devsecops-engineering-workflows-with-gitlab-duo",{"_path":4049,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4050,"content":4056,"config":4060,"_id":4062,"_type":13,"title":4063,"_source":15,"_file":4064,"_stem":4065,"_extension":18},"/en-us/blog/structuring-the-gitlab-package-registry-for-enterprise-scale",{"title":4051,"description":4052,"ogTitle":4051,"ogDescription":4052,"noIndex":6,"ogImage":4053,"ogUrl":4054,"ogSiteName":685,"ogType":686,"canonicalUrls":4054,"schema":4055},"Structuring the GitLab Package Registry for enterprise scale","Learn how to leverage GitLab's unique project-based publishing model alongside root-group-level consumption to create a secure, flexible package management strategy.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749662332/Blog/Hero%20Images/blog-image-template-1800x945__23_.png","https://about.gitlab.com/blog/structuring-the-gitlab-package-registry-for-enterprise-scale","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Structuring the GitLab Package Registry for enterprise scale\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tim Rizzi\"}],\n        \"datePublished\": \"2025-02-19\",\n      }",{"title":4051,"description":4052,"authors":4057,"heroImage":4053,"date":2932,"body":4058,"category":694,"tags":4059},[932],"As organizations grow, managing internal packages becomes increasingly\ncomplex. While traditional package managers, like JFrog Artifactory and\nSonatype Nexus, use a centralized repository approach, GitLab takes a\ndifferent path that aligns with modern development teams' work. In this\npost, we'll explore how to effectively structure your GitLab Package\nRegistry for enterprise scale, focusing on Maven and npm packages as\nexamples.\n\n\n## Understanding the GitLab Package Registry model\n\n\nIf you're coming from a traditional package manager, GitLab's approach might\ninitially seem different. Instead of a single centralized repository, GitLab\nintegrates package management directly into your existing project and group\nstructure. This means:\n\n\n- Teams publish packages to specific projects where the code lives\n\n- Teams consume packages from root group registries that aggregate all\npackages below them\n\n- Access control inherits from your existing GitLab permissions\n\n\nThis model offers several advantages:\n\n\n- Clear ownership of packages alongside their source code\n\n- Granular access control without additional configuration\n\n- Simplified CI/CD integration\n\n- Natural alignment with team structures\n\n- Single URL for accessing all company packages through root group\nconsumption\n\n\n### The power of root group package registry\n\n\nWhile GitLab supports package consumption at various group levels, using the\nroot group level has emerged as a best practice among our users. Here's why:\n\n\n- **Single access point:** One URL provides access to all private packages\nacross your organization\n\n- **Consistent package naming:** Group-level endpoints allow teams to\nmaintain their preferred naming conventions without conflicts\n\n- **Simplified configuration:** All developers can use the same\nconfiguration to access packages\n\n- **Secure access management:** Combines with deploy tokens for easy\nrotation and access control\n\n- **Hierarchical organization**: Naturally maps to your organizational\nstructure while maintaining unified access\n\n\n## Real-world example: Enterprise structure\n\n\nLet's look at how this works in practice with a large enterprise:\n\n\n```\n\ncompany/ (root group)\n\n├── retail-division/\n\n│   ├── shared-libraries/     # Division-specific shared code\n\n│   └── teams/\n\n│       ├── checkout/        # Team publishes packages here\n\n│       └── inventory/       # Team publishes packages here\n\n├── banking-division/\n\n│   ├── shared-libraries/    # Division-specific shared code\n\n│   └── teams/\n\n│       ├── payments/       # Team publishes packages here\n\n│       └── fraud/         # Team publishes packages here\n\n└── shared-platform/        # Enterprise-wide shared code\n    ├── java-commons/      # Shared Java libraries\n    └── ui-components/     # Shared UI components\n```\n\n\n### Publishing configuration\n\n\nTeams publish packages to their specific project registries, maintaining\nclear ownership:\n\n\n1. Maven example\n\n\n```xml\n\n\u003C!-- checkout/pom.xml -->\n\n\u003CdistributionManagement>\n    \u003Crepository>\n        \u003Cid>gitlab-maven\u003C/id>\n        \u003Curl>${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/maven\u003C/url>\n    \u003C/repository>\n\u003C/distributionManagement>\n\n```\n\n\n2. npm example\n\n\n```json\n\n// ui-components/package.json\n\n{\n  \"name\": \"@company/ui-components\",\n  \"publishConfig\": {\n    \"registry\": \"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/\"\n  }\n}\n\n```\n\n\n### Consuming configuration\n\n\nThe power of root group consumption comes into play here. All teams\nconfigure a single endpoint for package access:\n\n\n1. Maven example\n\n\n```xml\n\n\u003C!-- Any project's pom.xml -->\n\n\u003Crepositories>\n    \u003Crepository>\n        \u003Cid>gitlab-maven\u003C/id>\n        \u003Curl>https://gitlab.example.com/api/v4/groups/company/-/packages/maven\u003C/url>\n    \u003C/repository>\n\u003C/repositories>\n\n```\n\n\n2. npm example\n\n\n```\n\n# Any project's .npmrc\n\n@company:registry=https://gitlab.example.com/api/v4/groups/company/-/packages/npm/\n\n```\n\n\nThis configuration automatically provides access to all packages across your\norganization while maintaining the benefits of project-based publishing.\n\n\n## Authentication and access control\n\n\nGitLab's model simplifies authentication through deploy tokens and CI/CD\nintegration.\n\n\n### For CI/CD pipelines\n\n\nGitLab automatically handles authentication in pipelines using\n`CI_JOB_TOKEN`:\n\n\n```yaml\n\n# .gitlab-ci.yml\n\npublish:\n  script:\n    - mvn deploy  # or npm publish\n  # CI_JOB_TOKEN provides automatic authentication\n```\n\n\n### For development\n\n\nUse group deploy tokens for package consumption:\n\n\n- Create read-only deploy tokens at the root group level\n\n- Rotate tokens periodically for security\n\n- Share a single configuration across all developers\n\n\n## Benefits of root group package registry\n\n\n1. Simplified configuration\n   - One URL for all package access\n   - Consistent setup across teams\n   - Easy token rotation\n2. Clear ownership\n   - Packages stay with their source code\n   - Teams maintain control over publishing\n   - Version history tied to project activity\n3. Natural organization\n   - Matches your company structure\n   - Supports team autonomy\n   - Enables cross-team collaboration\n\n## Getting started\n\n\n1. Set up your root group\n   - Create a clear group structure\n   - Configure appropriate access controls\n   - Create group deploy tokens\n2. Configure team projects\n   - Set up project-level publishing\n   - Implement CI/CD pipelines\n   - Document package naming conventions\n3. Standardize consumption\n   - Configure root group registry access\n   - Share deploy tokens securely\n   - Document package discovery process\n\n## Summary\n\n\nGitLab's package registry model, particularly when leveraging root group\nconsumption, offers a powerful solution for enterprise package management.\nBy combining project-based publishing with root group consumption,\norganizations get the best of both worlds: clear ownership and simplified\naccess. This approach scales naturally with your organization while\nmaintaining security and ease of use.\n\n\nStart by implementing this model with a single team or division, and expand\nas you see the benefits of this integrated approach. Remember that while\nthis post focused on Maven and npm, the same principles apply to all package\ntypes supported by GitLab.\n\n\n> Get started with package registries today! Sign up for a [free\ntrial of GitLab Ultimate](https://about.gitlab.com/free-trial/).\n",[851,9,1016],{"slug":4061,"featured":90,"template":700},"structuring-the-gitlab-package-registry-for-enterprise-scale","content:en-us:blog:structuring-the-gitlab-package-registry-for-enterprise-scale.yml","Structuring The Gitlab Package Registry For Enterprise Scale","en-us/blog/structuring-the-gitlab-package-registry-for-enterprise-scale.yml","en-us/blog/structuring-the-gitlab-package-registry-for-enterprise-scale",{"_path":4067,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4068,"config":4071,"content":4073,"_id":4078,"_type":13,"title":4079,"_source":15,"_file":4080,"_stem":4081,"_extension":18},"/en-us/blog/supercharge-your-git-workflows",{"title":4069,"description":4070},"Supercharge your Git workflows","Optimize git clone operations in any environment — up to 93% reduction in clone times and 98% reduction in disk space usage — with the Git Much Faster script.",{"slug":4072,"featured":90,"template":700},"supercharge-your-git-workflows",{"title":4069,"description":4070,"authors":4074,"heroImage":2613,"date":4075,"category":785,"tags":4076,"body":4077},[691],"2025-09-10",[9,742],"Picture this: You're working on the Chromium project and you need to clone the repository. You run `git clone`, grab a coffee, check your email, maybe take a lunch break, and 95 minutes later, you finally have your working directory. This is the reality for developers working with large repositories containing 50GB+ of data.\n\nThe productivity impact is staggering. CI/CD pipelines grind to a halt waiting for repository clones. Infrastructure costs skyrocket as compute resources sit idle. Developer frustration mounts as context-switching becomes the norm.\n\nBut what if that 95-minute wait could be reduced to just 6 minutes? What if you could achieve a 93% reduction in clone times using proven techniques?\n\nEnter [Git Much Faster](https://gitlab.com/gitlab-accelerates-embedded/misc/git-much-faster) — a comprehensive benchmarking and optimization script that transforms how you work with large Git repositories. Built from real-world experience optimizing embedded development workflows, this script provides practical strategies delivering measurable performance improvements across standard git clones, optimized configurations, and Git's built-in Scalar tool.\n\nYou'll discover how to dramatically reduce git clone times using optimization strategies, explore real-world performance benchmarks from major repositories like the Linux kernel and Chromium, and understand how to implement these optimizations safely in both development and CI/CD environments.\n\n## Project overview: What is Git Much Faster?\n\nGit Much Faster is a script I wrote as an enablement tool to allow you to benchmark multiple clone optimization approaches on the same client — whether that is a traditional developer workstation, CI, cloud-hosted development environments or specialized clones for GitOps. It also contains the curated configuration settings for the fastest clone optimization. You can use these settings as a starting point and adapt or remove configurations that create too lean of a clone for your client's intended use of the repository clone.\n\nGit Much Faster addresses a fundamental challenge: Git's default clone behavior prioritizes safety over speed. While this works for small repositories, it becomes a significant bottleneck with large codebases, extensive binary assets, or complex monorepo structures.\n\nThe problem manifests across increasingly common scenarios. Embedded development teams inherit repositories filled with legacy firmware binaries, bootloaders, and vendor SDKs stored directly in version control. Web applications accumulate years of marketing assets and design files. Game development projects contain massive 3D models and audio files growing repository sizes into tens of gigabytes.\n\nEnterprise CI/CD pipelines suffer particularly acute pain. Each job requires a fresh repository clone, and when operations take 20 to 90 minutes, entire development workflows grind to a halt. Infrastructure costs multiply as compute resources remain idle during lengthy clone operations.\n\nGit Much Faster solves this through comprehensive benchmarking comparing four distinct strategies: standard git clone (baseline with full history), optimized git clone (custom configurations with compression disabled and sparse checkout), Git's Scalar clone (integrated partial cloning), and current directory assessment (analyzing existing repositories without re-cloning).\n\nThe tool provides measurable, repeatable benchmarking in controlled AWS environments, eliminating variables that make performance testing unreliable. The real power of Git Much Faster is to run all the benchmarks in whatever your target environment looks like — so if slow network connections are a reality for some developers, you can decipher the best clone optimization for their situation.\n\n## Technical deep dive: The optimization strategies\n\nUnderstanding Git Much Faster's effectiveness requires examining specific configurations that address Git's performance bottlenecks through a layered approach tackling network transfer efficiency, CPU utilization, and storage patterns.\n\nThe most significant gains come from two key optimizations. The first, `core.compression=0`, eliminates CPU-intensive compression during network operations. CPU cycles spent compressing often exceed bandwidth savings on modern high-speed networks. This optimization alone reduces clone times by 40% to 60%.\n\nThe second major optimization, `http.postBuffer=1024M`, addresses Git's conservative HTTP buffer sizing. Large repositories benefit tremendously from increased buffer sizes, allowing Git to handle larger operations without breaking them into multiple requests, reducing protocol overhead.\n\nGit Much Faster leverages shallow clones using `--depth=1` (fetching only the latest commit) and partial clones with `--filter=blob:none` (deferring file content downloads until checkout). Shallow clones reduce data by 70%-90% for mature repositories, while partial clones prove particularly effective for repositories with large binary assets.\n\nSparse checkout provides surgical precision in controlling checked-out files. Git Much Faster implements comprehensive exclusion covering 30+ binary file types — images, documents, archives, media files, and executables — reducing working directory size by up to 78% while maintaining full source code access.\n\nGit's Scalar tool, integrated into Git since Version 2.38, combines partial clone, sparse checkout, and background maintenance. However, benchmarking reveals Scalar doesn't implement the aggressive compression and buffer optimizations providing the most significant performance gains. Testing shows the custom optimized approach typically outperforms Scalar by 48%-67% while achieving similar disk space savings.\n\n## End-to-end load reduction\n\nAn interesting thing about optimizing the clone operation is that it also reduces complete system loading because you are reducing the size of your request. GitLab has a specialized, horizontal scaling layer known as [Gitaly Cluster](https://docs.gitlab.com/administration/gitaly/praefect/). When full history clones and large monorepos are the norm, the sizing of Gitaly Cluster is driven higher. This is because all git clone requests are serviced by a server-side binary to create “pack files” to be sent over the wire. Since these server-side git operations involve running compression utilities, it drives all three of memory, CPU, and I/O requirements at once.\n\nWhen git clone operations are optimized to reduce the size of the total content ask, it reduces load on the end-to-end stack: Client, Network, Gitaly Service and Storage. All layers speed up and become cheaper at the same time.\n\n## Real-world performance results\n\nGit Much Faster's effectiveness is demonstrated through rigorous benchmarking across diverse, real-world repositories using consistent AWS infrastructure with Arm instances and controlled network conditions.\n\n**Linux kernel repository (7.5GB total):** Standard clone took 6 minutes 29 seconds. Optimized clone achieved 46.28 seconds — an 88.1% improvement, reducing the .git directory from 5.9GB to 284MB. Scalar took 2 minutes 21 seconds (63.7% improvement), completing 67.3% slower than the optimized approach.\n\n**Chromium repository (60.9GB total):** Standard clone required 95 minutes 12 seconds. Optimized clone achieved 6 minutes 41 seconds — a dramatic 93% improvement, compressing the .git directory from 55.7GB to 850MB. Scalar took 13 minutes 3 seconds (86.3% improvement) but remained 48.8% slower than the optimized approach.\n\n**GitLab website repository (8.9GB total):** Standard clone took 6 minutes 23 seconds. Optimized clone achieved 6.49 seconds — a remarkable 98.3% improvement, reducing the .git directory to 37MB. Scalar took 33.60 seconds (91.2% improvement) while remaining 80.7% slower.\n\nThe benchmarking reveals clear patterns: Larger repositories show more dramatic improvements, binary-heavy repositories benefit most from sparse checkout filtering, and the custom optimization approach consistently outperforms both standard Git and Scalar across all repository types.\n\n## Practical implementation guide\n\nImplementation requires understanding when to apply each technique based on use case and risk tolerance. For development requiring full repository access, use standard Git cloning. For read-heavy workflows needing rapid access to current code, deploy optimized cloning. For CI/CD pipelines where speed is paramount, optimized cloning provides maximum benefit.\n\nGetting started requires only simple download and execution:\n\n```bash\ncurl -L https://gitlab.com/gitlab-accelerates-embedded/misc/git-much-faster/-/raw/master/git-much-faster.sh -o ./git-much-faster.sh\n\n# For benchmarking\nbash ./git-much-faster.sh --methods=optimized,standard --repo=https://github.com/your-org/your-repo.git\n```\n\nFor production-grade testing, Git Much Faster project includes complete Terraform infrastructure for AWS deployment, eliminating variables that skew local testing results.\n\nOptimized clones require careful consideration of limitations. Shallow clones prevent access to historical commits, limiting operations like `git log` across file history. For teams adopting optimizations it is best to create specific optimizations for high volume usage. For instance, developers can perform an optimized clone, and if and when needed, convert to full clones when needed via `git fetch --unshallow`. If a given CI job accesses commit history (e.g. using GitVersion), then you may need the full history, but not a checkout.\n\n## Use cases and industry applications\n\nEmbedded development presents unique challenges where projects historically stored compiled firmware and hardware design files directly in version control. These repositories often contain FPGA bitstreams, PCB layouts, and vendor SDK distributions ballooning sizes into tens of gigabytes. Build processes frequently require cloning dozens of external repositories, multiplying performance impact.\n\nEnterprise monorepos encounter Git performance challenges as repositories grow encompassing multiple projects and accumulated historical data. Media and asset-heavy projects compound challenges, as mentioned above — web applications accumulate marketing assets over years, while game development faces severe challenges with 3D models and audio files pushing repositories beyond 100GB. [More use cases](https://gitlab.com/gitlab-accelerates-embedded/misc/git-much-faster/-/tree/master?ref_type=heads#common-use-cases) can be found in the project.\n\nCI/CD pipelines represent the most impactful application. Each container-based CI job requires a fresh repository clone, and when operations consume 20 to 90 minutes, entire development workflows become unviable.\n\nGeographically spread out development teams may have team members whose network performance to their primary development workstation is extremely limited or varies dramatically. Optimizing the Git clone can help by reducing over the wire sizes dramatically.\n\n## Next steps\n\nGit clone optimization represents a transformative opportunity delivering measurable improvements — up to 93% reduction in clone times and 98% reduction in disk space usage — that fundamentally change how teams interact with codebases.\n\nThe key insight is that Git's default conservative approach leaves substantial performance opportunities untapped. By understanding specific bottlenecks — network transfer inefficiency, CPU-intensive compression, unnecessary data downloads — teams can implement targeted optimizations delivering transformative results.\n\n\n**Ready to revolutionize your Git workflows?**\n\n[Read the docs in the Git Much Faster repository](https://gitlab.com/gitlab-accelerates-embedded/misc/git-much-faster) and get started running benchmarks against your largest repositories. Begin with read-only optimization in CI/CD pipelines where benefits are immediate and risks minimal. As your team gains confidence, gradually expand optimization to development workflows based on measured results.\n\nThe future of Git performance optimization continues evolving, but fundamental principles — eliminating unnecessary work, optimizing for actual bottlenecks, measuring results rigorously — remain valuable regardless of future tooling evolution. Teams mastering these concepts today position themselves to leverage whatever improvements tomorrow's Git ecosystem provides.\n","content:en-us:blog:supercharge-your-git-workflows.yml","Supercharge Your Git Workflows","en-us/blog/supercharge-your-git-workflows.yml","en-us/blog/supercharge-your-git-workflows",{"_path":4083,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4084,"content":4089,"config":4095,"_id":4097,"_type":13,"title":4098,"_source":15,"_file":4099,"_stem":4100,"_extension":18},"/en-us/blog/take-advantage-of-git-rebase",{"title":4085,"description":4086,"ogTitle":4085,"ogDescription":4086,"noIndex":6,"ogImage":4016,"ogUrl":4087,"ogSiteName":685,"ogType":686,"canonicalUrls":4087,"schema":4088},"Take advantage of Git rebase","Tap into the Git rebase features to improve your workflow.","https://about.gitlab.com/blog/take-advantage-of-git-rebase","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Take advantage of Git rebase\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Christian Couder\"}],\n        \"datePublished\": \"2022-10-06\",\n      }",{"title":4085,"description":4086,"authors":4090,"heroImage":4016,"date":4092,"body":4093,"category":785,"tags":4094},[4091],"Christian Couder","2022-10-06","These days, developers spend a lot of time reviewing merge requests\n\nand taking these reviews into account to improve the code. We'll discuss how\n\n[Git rebase](https://git-scm.com/docs/git-rebase) can help in\n\nspeeding up these review cycles. But first, let's take a look at some\n\nworkflow considerations.\n\n\n## Different ways to rework a merge request\n\n\nA developer who worked on some code changes and created a merge\n\nrequest with these changes will often have to rework them. Why does\n\nthis happen? Tests can fail, bugs are found, or reviewers suggest\n\nimprovements and find shortcomings.\n\n\n### Simple but messy method: add more commits\n\n\nOne way to rework the code changes is to make more changes in some new\n\ncommits on top of the branch that was used to create the merge\n\nrequest, and then push the branch again to update the merge\n\nrequest.\n\n\nWhen a number of commits have been added in this way, the merge\n\nrequest becomes problematic:\n\n\n- It's difficult to review by looking at all the changes together.\n\n- It's difficult to review the commits separately as they may contain\ndifferent unrelated changes, or even multiple reworks of the same code.\n\n\nReviewers find it easier to review changes split into a number of small,\n\nself-contained commits that can be reviewed individually.\n\n\n### Pro method: rebase!\n\n\nA better method to prepare or rework a merge request is to always\n\nensure that each commit contains small, self-contained, easy-to-review\n\nchanges.\n\n\nThis means that all the commits in the branch may need reworking\n\ninstead of stacking on yet more commits. This approach might seem much\n\nmore complex and tedious, but `git rebase` comes to the rescue!\n\n\n## Rework your commits with `git rebase`\n\n\nIf your goal is to build a merge request from a series of small,\n\nself-contained commits, your branch may need significant rework before its\n\ncommits are good enough. When the commits are ready, you can push the branch\n\nand update or create a merge request with this branch.\n\n\n### Start an interactive rebase\n\n\nIf your branch is based on `main`, the command to rework your branch\n\nis:\n\n\n```plaintext\n\ngit rebase -i main\n\n```\n\n\nI encourage you to create [a Git\nalias](https://git-scm.com/book/en/v2/Git-Basics-Git-Aliases),\n\nor a shell alias or function for this command right away, as you will\n\nuse it very often.\n\n\nThe `-i` option passed to `git rebase` is an alias for\n\n`--interactive`. It starts\n\n[an 'interactive'\nrebase](https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---interactive)\n\nwhich will open your editor. In it, you will find a list of the\n\ncommits in your branch followed by commented-out lines beginning with\n\n`#`. The list of commits looks like this:\n\n\n```plaintext\n\npick 1aac632db2 first commit subject\n\npick a385014ad4 second commit subject\n\npick 6af12a88cf other commit subject\n\npick 5cd121e2a1 last commit subject\n\n```\n\n\nThese lines are instructions for how `git rebase` should handle these\n\ncommits. The commits are listed in chronological order, with the\n\noldest commit at the top. (This order is the opposite of the default\n\n`git log` order.) What do these lines contain?\n\n\n- An instruction (here, `pick`) that tells Git what action to take\n\n- An abbreviated commit ID\n\n- A commit subject to help you identify the commit contents\n\n\n### Edit the instruction list\n\n\nYou can edit these instructions! When you quit your text editor, `git\nrebase`\n\nreads the instructions you've just edited, and performs them\n\nin sequence to recreate your branch the way you want.\n\n\nAfter the instructions for all commits, a set of commented-out lines\n\nexplain how to edit the instruction lines, and how each instruction\n\nwill change your branch:\n\n\n- If you **delete a commit's entire instruction line** from the list,\n  that commit won't be recreated.\n- If you **reorder the instruction lines**, the commits will be\n  recreated in the order you specify.\n- If you **change the action** from `pick` to something else, such as\n  `squash` or `reword`, Git performs the action you specify on that\n  commit.\n- You can even **add new instruction lines** before, after, or between\n  existing lines.\n\nIf the comment lines aren't enough, more information about what you\n\ncan do and how it works is available in:\n\n\n- The [Git Tools - Rewriting\nHistory](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History)\n  section of the \"Pro Git\" book\n- The [Interactive\nmode](https://git-scm.com/docs/git-rebase#_interactive_mode)\n  section of the `git rebase` documentation\n\n### Continue or abort the rebase\n\n\nAn interactive rebase can stop if there is a conflict (as a regular\n\nrebase would) or if you used an instruction like `edit` in the\n\ninstruction line. This allows you to make some changes, like splitting\n\nthe current commit into two commits, or fixing the rebase conflict if\n\nthere is one. You can then either:\n\n\n- Continue the interactive rebase with `git rebase --continue`.\n\n- Abort the rebase altogether with `git rebase --abort`.\n\n\n(These `git rebase` options also work when a regular, non-interactive\n\nrebase stops.)\n\n\n## Further tips and benefits\n\n\n### Try different instructions\n\n\nI recommend you try out the different instructions you can use in\n\neach instruction line, especially `reword`, `edit`, `squash`, and `fixup`.\nYou'll\n\nsoon want to use the abbreviated versions of these instructions: `r`,\n\n`e`, `s`, and `f`.\n\n\n### Run shell commands in your rebase\n\n\nYou might also have noticed an `exec \u003Ccommand>` instruction that\n\nallows you to run any shell command at any point in the interactive rebase.\n\nI've found it more useful for non-interactive rebases, such as:\n\n\n```plaintext\n\ngit rebase --exec 'make test' main\n\n```\n\n\n(It's not an interactive rebase because it doesn't contain the `-i` flag.)\n\n\nThe `--exec \u003Ccommand>` flag allows you to run any shell command after\n\neach rebased commit, stopping if the shell command fails (which is\n\nsignaled by a non zero exit code).\n\n\n### Test all your commits\n\n\nPassing a command to build your software and run its tests, like\n\n`make test`, to `--exec` will check that each commit in your branch\n\nbuilds correctly and passes your tests.\n\n\nIf `make test` fails, the rebase stops. You can then fix the current\n\ncommit right away, and continue the rebase to test the next\n\ncommits.\n\n\nChecking each commit builds cleanly and passes all the tests ensures\n\nyour code base is always in a good state. It's especially useful if\n\nyou want to take advantage of\n\n[Git bisect](https://git-scm.com/docs/git-bisect) when you encounter\n\nregressions.\n\n\n## Conclusion\n\n\nIn Git, a rebase is a very versatile and useful tool to rework\n\ncommits. Use it to achieve a workflow with high-quality changes\n\nproposed in high-quality commits and merge requests. It makes your\n\ndevelopers and reviewers more efficient. Code reviews and debugging also\nbecome easier and more effective.\n\n\n**EDIT:** Check out our [follow-up post on how you can apply this is real\nlife](/blog/rebase-in-real-life/).\n",[742,956,893,9],{"slug":4096,"featured":6,"template":700},"take-advantage-of-git-rebase","content:en-us:blog:take-advantage-of-git-rebase.yml","Take Advantage Of Git Rebase","en-us/blog/take-advantage-of-git-rebase.yml","en-us/blog/take-advantage-of-git-rebase",{"_path":4102,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4103,"content":4109,"config":4114,"_id":4116,"_type":13,"title":4117,"_source":15,"_file":4118,"_stem":4119,"_extension":18},"/en-us/blog/the-ultimate-guide-to-enabling-saml",{"title":4104,"description":4105,"ogTitle":4104,"ogDescription":4105,"noIndex":6,"ogImage":4106,"ogUrl":4107,"ogSiteName":685,"ogType":686,"canonicalUrls":4107,"schema":4108},"The ultimate guide to enabling SAML and SSO on GitLab.com","Learn how to make full use of SAML and SSO security features on the GitLab DevSecOps platform.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666495/Blog/Hero%20Images/cover-1800x945.png","https://about.gitlab.com/blog/the-ultimate-guide-to-enabling-saml","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The ultimate guide to enabling SAML and SSO on GitLab.com\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Bradley Lee\"}],\n        \"datePublished\": \"2023-09-14\",\n      }",{"title":4104,"description":4105,"authors":4110,"heroImage":4106,"date":1858,"body":4112,"category":830,"tags":4113},[4111],"Bradley Lee","\nAs a follow-on to the recent blog, [The ultimate guide to securing your code on\nGitLab.com](https://about.gitlab.com/blog/securing-your-code-on-gitlab/),\nwe recommended enabling SAML (Security Assertion Markup Language) and SSO (single\nsign-on) to enable tighter control over code access. Let’s take a deep dive into\nhow to enable SAML and SSO on GitLab.com.\n\n## What are SAML and SSO?\nSAML is an open standard, which service providers (like GitLab.com) and\nidentity providers (commonly referred to as IdPs) use to communicate\nauthentication data. SSO is provided by IdPs, such as Okta and Entra ID\n(formerly Azure AD), and enables users to log into multiple systems or service\nproviders through a single interface with a single set of credentials.\n\nAs with any configuration, there should be thoughtful and careful planning when\nenabling SSO.\n\n### What are the benefits of SSO?\nIn general, enabling SSO streamlines the user experience by unifying the login\nprocess and reducing the account and password bloat required for multiple\nenterprise applications. Enabling SSO also adds an extra layer of security and\nmanagement efficiency for identity management teams by providing a single\nsource of truth for authentication. Below, you’ll learn how SAML SSO applies\nspecifically to GitLab.com.\n\n## Configuring SSO and SAML for GitLab.com\nPremium and Ultimate tiers can enable SSO in the settings available at the\nnamespace or top level group.\n\n### Enabling SSO at the group level\nBefore getting started, you’ll need a few key\npieces of information from your chosen IdP:\n- The IdP SSO URL\n- The certificate fingerprint provided by the IdP application\n\nOnce these key pieces are entered, check the “Enable SAML\nauthentication for this group” box. \n\n### How user accounts are linked\nBefore we proceed further into configuration, let’s take a look at how GitLab\nauthenticates against the IdP.\n\nFor GitLab.com, each user who requires access to\nthe system must have an account on GitLab.com. By default, when a user first\nattempts logging into GitLab via SSO, GitLab will receive the SAML assertion\nand validate if the identity (specifically the email address) is linked to a\nGitLab.com account. If not, GitLab will request the user either login to an\nexisting account or create a new account. In most instances, this may not be\ndesired behavior; however, we will address this later in the process. We’ve\nprovided a flowchart below to help you navigate the provisioning flow.\n\n![image of saml group links](https://about.gitlab.com/images/blogimages/2023-09-14-ultimate-guide-to-enabling-saml/saml-provisioning.png)\n\n### Enforcing SSO\nTo further increase security, there are two options available for enforcing\nSSO. Assuming neither are checked, users with access to the namespace can log\nin with either the SSO credentials or the GitLab.com credentials.\n\nHere is a working example that we can use to follow along as we discuss how the\nconfiguration options affect our baseline. Let’s consider a user in the IdP\nwhere the username is `idpusername` and contains a super secret password:\n`idppassword` (apologies, security professionals). Taking into account the\ninformation we just learned about account linking, let us also assume our demo\nuser created a new account following the prompt from an SSO login with a\nusername of `gitlabusername` and `gitlabpassword` as an even more secure\npassword.\n\n#### Enforcing SSO only for web\nWhen enabling the “Enforce SSO-only authentication for web activity for this\ngroup” setting, all members must now access all groups and projects under the\nhierarchy using the configured SSO login regardless of whether they have an\nexisting SAML identity. As we mentioned prior, with this flag disabled, our\n`idpusername` user will be able to log into the GitLab namespace with either\nthe `idpusername` or `gitlabusername` credential sets. When we enable this\nsetting for web-based activity ([further details in\ndocs](https://docs.gitlab.com/ee/user/group/saml_sso/#sso-only-for-web-activity-enforcement)),\nour group is now only accessible by the `idpusername` credential set.\n\n#### Enforcing SSO only for Git proxy\nVery similar to enforcing SSO for web, when the “Enforce SSO-only\nauthentication for Git and Dependency Proxy” activity for this group option is\nenabled, a few things happen:\n- Calling an API endpoint that involves Git activity requires SSO.\n- For Git activity over SSH and HTTPS, users must have at least one active session signed-in through SSO before they can push to or pull.\n\nThere is a strong recommendation to enable both of these settings to take full\nadvantage of the benefits of SSO for users and administrators through\ncentralized authentication.\n\n### Enterprise user support\nNow that we know how some of the configuration options can help secure access,\nlet’s take a deeper dive into user management. Consider the following scenario:\nOur `idpusername` user has decided to pursue another opportunity outside of the\ndomain. Based on what we have configured now, once the account has been\ndeprovisioned from the IdP, it should no longer have access to anything secured\nbehind it on GitLab.com. However, while the user will not have access, the\nassociated user ID and roles still remain until manually removed. This is where\nEnterprise users come in.\n\n#### What are Enterprise users in GitLab?\nIf you look closely, any user that has a linked SSO account will carry a `SAML`\nbadge in the member list. GitLab also has an associated `Enterprise` badge\nthat grants additional management functionality through SSO. For a user to\ncarry the `Enterprise` badge, the user must either have the initial GitLab.com account creation initiated by a SAML SSO login or have the initial GitLab.com account created by SCIM.\n\n#### What is SCIM?\nSCIM, or System for Cross-domain Identity Management, is another standard\nused in conjunction with SAML, primarily for provisioning and deprovisioning\nacross multiple systems. By enabling SCIM for your GitLab.com group (which is\ncurrently supported with Entra ID and Okta), you can enable automatic\nprovisioning and deprovisioning of accounts.\n\nIf we look back at some of our scenarios, without SCIM, our `idpusername` user\nwas prompted to create or link a GitLab.com account on first login. With SCIM\nenabled, this process is handled automatically based on information provided\nand managed by the IdP and is completely transparent to the end user. The\nsecond half of our scenario, where our `idpusername` user is deprovisioned from\nthe IdP, also is solved with automation via SCIM. In this instance, when the\nuser is removed on the IdP side, SCIM automatically disconnects the SAML\nidentity from the GitLab.com account and removes the user from the GitLab.com\ngroup.\n\n#### Protecting your intellectual property\nAnother important feature of Enterprise users is the ability to control two\nvery important user settings that are not accessible to group administrators on\nGitLab.com. Since all users require an account on GitLab.com, they are also\ngranted access to a personal user namespace. For example, our `idpusername` will have access to our Acme Corp. group at `.com/acmecorp`, and will also have\naccess to their own personal space at `.com/idpusername`. One common concern with this is the ability for users to take code out of the organization\nnamespace and commit to their own personal namespace.\n\nWith Enterprise users, we have two settings that we can control based on attributes received in the SAML\nresponse. These keys are `projects_limit` and `can_create_group`. The\n`projects_limit` is an integer value that sets the amount of projects a user\ncan create in their personal namespace. When set to `0`, this effectively\ndisables project creation in that space. Similarly, `can_create_group` is a\nboolean `true` or `false` value that indicates whether a user can create new\ngroups.\n\n### Managing roles with SAML\nNow that we know the ins and outs of creating and removing users with SAML and\nSCIM, how can we leverage our work to help manage our active users? In this\nfinal section, we’ll take a look at why we recommend setting default membership\nto \"Minimal Access\" and how to leverage group memberships in the IdP.\n\n#### Why Minimal Access?\nIn the [Ultimate guide to securing your code on GitLab](https://about.gitlab.com/blog/securing-your-code-on-gitlab/),\nwe recommend setting the default membership role to Minimal Access, and\noperating with the concept of least privilege. Roles can be elevated as needed\nin subgroups or individual projects while preventing visibility to projects or\nsubgroups where the user is not explicitly granted another role. By default,\nthis option is set to Guest, which will allow all provisioned users guest\naccess to the repositories. Default membership controls are available at the\ntop-level group, along with the SAML and SSO settings. For automation at the\nsubgroup level, we can leverage SAML Group Sync.\n\n#### Configuring SAML Group Sync with SAML Group Links\nBefore we dive into the configuration, there is one very important step we need\nto take. The configured SAML assertion that is sent MUST include an attribute\nnamed `Groups` or `groups`. If SAML Group Links are present without the\nattribute in the assertion, users may be removed from the group or reverted to\nMinimal Access.\n\nAfter we ensure our assertions contain the necessary information, we can start\nusing SAML Group Links to automatically assign membership roles to GitLab\ngroups based on group membership in the IdP. Let’s build on our demo user\n`idpusername` by considering the following:\n- `idpusername` is a maintainer on the acme-web project.\n- The `acme-web` project exists under the `acme-corp` namespace, under subgroup `acme-com`.\n- The full path to the project would be `.com/acme-corp/acme-com/acme-web`.\n- `idpusername` should also be granted developer access for the `acme-db` project, which is also under the `acme-com` group.\n- In our IdP, `idpusername` is a member of the IdP group `idp-acme-com`.\n\nSAML group links allow us to map IdP group memberships to role assignments at\nthe GitLab group level. In this scenario, we can create a group link at the\n`acme-com` group in GitLab that maps the IdP group `idp-acme-com` to the\ndeveloper role to the `acme-com` group.\n\nDue to inheritance, our `idpusername`\nuser will be granted developer access and associated visibility to every\nproject and group that falls under the GitLab `acme-com` group automatically by\nvirtue of the IdP group membership, because we’re working under the concept of\nleast privilege for the `acme-web` project.\n\nThe `idpusername` user’s role can\nbe elevated to maintainer directly in the project. From a user perspective,\n`idpusername` would still carry the Minimal Access role at the `acme-corp`\ngroup as well. This allows a separation of access management between\nengineering and identity management teams and allows role management to be\nflexible with guardrails.\n\n![image of saml group links](https://about.gitlab.com/images/blogimages/2023-09-14-ultimate-guide-to-enabling-saml/saml-group-links.png)\n\nWith this approach, it’s important to find that balance between what is managed\nin the IdP and what is managed in GitLab. It’s possible to have hundreds of\ngroup mappings to roles in the IdP and almost completely remove role management\nwithin GitLab and vice versa. The flexibility that GitLab allows enables you to\nfind the best solution that works for you. Building on our example, if we hire\nanother engineer for the `acme-com` project, they can be added to the GitLab\napplication in the IdP, and added to the `idp-acme-com` group. This\nautomatically assigns them the developer role at the `acme-com` group and for\nall projects under it, while limiting access to any other groups outside of\n`acme-com` in the namespace.\n\n## Learn more\nWe’ve covered how to get started with enabling SAML and SSO on your GitLab.com\ngroup, along with how to leverage the features to programmatically manage users\nand roles with real examples. For more information, see the full [SAML SSO for\nGitLab.com groups](https://docs.gitlab.com/ee/user/group/saml_sso/)\ndocumentation.\n\nCover image by [Towfiqu barbhuiya](https://unsplash.com/photos/FnA5pAzqhMM) on [Unsplash](https://unsplash.com)\n{: .note}\n",[830,9,495],{"slug":4115,"featured":6,"template":700},"the-ultimate-guide-to-enabling-saml","content:en-us:blog:the-ultimate-guide-to-enabling-saml.yml","The Ultimate Guide To Enabling Saml","en-us/blog/the-ultimate-guide-to-enabling-saml.yml","en-us/blog/the-ultimate-guide-to-enabling-saml",{"_path":4121,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4122,"content":4127,"config":4132,"_id":4134,"_type":13,"title":4135,"_source":15,"_file":4136,"_stem":4137,"_extension":18},"/en-us/blog/the-ultimate-guide-to-gitops-with-gitlab",{"title":4123,"description":4124,"ogTitle":4123,"ogDescription":4124,"noIndex":6,"ogImage":2259,"ogUrl":4125,"ogSiteName":685,"ogType":686,"canonicalUrls":4125,"schema":4126},"The ultimate guide to GitOps with GitLab","This eight-part tutorial series demonstrates how to use GitLab as a best-in-class GitOps tool.","https://about.gitlab.com/blog/the-ultimate-guide-to-gitops-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The ultimate guide to GitOps with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Viktor Nagy\"}],\n        \"datePublished\": \"2022-04-07\",\n      }",{"title":4123,"description":4124,"authors":4128,"heroImage":2259,"date":4129,"body":4130,"category":785,"tags":4131},[2244],"2022-04-07","\n\nIt is possible to use GitLab as a best-in-class GitOps tool, and this blog post series is going to show you how. [GitOps](/topics/gitops/) is an operational framework that takes DevOps best practices used for application development such as version control, collaboration, compliance, and CI/CD tooling, and applies them to infrastructure automation. This series of easy-to-follow tutorials will focus on different user problems, including provisioning, managing a base infrastructure, and deploying various third-party or custom applications on top of them, that can be solved by pairing GitOps with GitLab.\n\nHere are 8 tutorials on how to do GitOps with GitLab:\n\n**1. [Here's how to do GitOps with GitLab](/blog/gitops-with-gitlab/)**\n\nThis tutorial sets the stage for what you will learn throughout the series, including the tech concepts you'll need to know.\n\n**2. [Infrastructure provisioning with GitLab and Terraform](/blog/gitops-with-gitlab-infrastructure-provisioning/)**\n\nThis tutorial walks you through setting up the underlying infrastructure using GitLab and Terraform.\n\n**3. [Connect with a Kubernetes cluster](/blog/gitops-with-gitlab-connecting-the-cluster/)**\n\nThis tutorial demonstrates how to connect a Kubernetes cluster with GitLab for pull- and push-based deployments and easy security integrations.\n\n**4. [How to tackle secrets management](/blog/gitops-with-gitlab-secrets-management/)**\n\nThis tutorial builds on the previous tutorial to show you how to use a Kubernetes cluster connection to manage secrets within a cluster.\n\n**5. [The CI/CD tunnel](/blog/gitops-with-gitlab-using-ci-cd/)**\n\nThis tutorial introduces you to CI/CD tunnels and shows step-by-step how to access a Kubernetes cluster using GitLab CI/CD.\n\n**6. [Connecting GitLab with a Kubernetes cluster - Auto DevOps](/blog/gitops-with-gitlab-auto-devops/)**\n\nThis tutorial looks at how you can use Auto DevOps with all its bells and whistles to easily manage deployments.\n\n**7. [Connecting GitLab with a Kubernetes cluster for GitOps-style application delivery](/blog/gitops-with-gitlab/)**\n\nThis tutorial shows you how to connect an application project to a manifest project for controlled, GitOps-style deployments.\n\n**8. [Turn a GitLab agent for Kubernetes installation to manage itself](/blog/gitops-with-gitlab-manage-the-agent/)**\n\nThis tutorial is the culmination of the previous tutorials and will teach you how to turn a GitLab agent for Kubernetes installation to manage itself.\n\n\n**Read more about GitOps:**\n- [GitLab for GitOps](/solutions/gitops/)\n- [What is GitOps](/topics/gitops/)\n- [GitOps viewed as part of the Ops evolution](/blog/gitops-as-the-evolution-of-operations/)\n- [How to use a push-based approach for GitOps with GitLab scripting and variables](/blog/how-to-agentless-gitops-vars/)\n\n\n\n\n",[789,9,549],{"slug":4133,"featured":6,"template":700},"the-ultimate-guide-to-gitops-with-gitlab","content:en-us:blog:the-ultimate-guide-to-gitops-with-gitlab.yml","The Ultimate Guide To Gitops With Gitlab","en-us/blog/the-ultimate-guide-to-gitops-with-gitlab.yml","en-us/blog/the-ultimate-guide-to-gitops-with-gitlab",{"_path":4139,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4140,"content":4146,"config":4151,"_id":4153,"_type":13,"title":4154,"_source":15,"_file":4155,"_stem":4156,"_extension":18},"/en-us/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab",{"title":4141,"description":4142,"ogTitle":4141,"ogDescription":4142,"noIndex":6,"ogImage":4143,"ogUrl":4144,"ogSiteName":685,"ogType":686,"canonicalUrls":4144,"schema":4145},"The ultimate guide to least privilege access with GitLab","This tutorial demonstrates how to achieve least privilege access using custom roles, security policies, compliance pipelines, branch protections, and more.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099438/Blog/Hero%20Images/Blog/Hero%20Images/built-in-security_built-in-security.jpeg_1750099438377.jpg","https://about.gitlab.com/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The ultimate guide to least privilege access with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2024-03-06\",\n      }",{"title":4141,"description":4142,"authors":4147,"heroImage":4143,"date":4148,"body":4149,"category":830,"tags":4150},[1444],"2024-03-06","The principle of least privilege\n([PoLP](https://csrc.nist.gov/glossary/term/least_privilege)) is a concept\nin which a user's access rights should be limited to the bare minimum needed\nfor them to complete the tasks required within their respective roles. By\nimplementing PoLP you can enhance your organization's [security\nposture](https://csrc.nist.gov/glossary/term/security_posture),\ncomplementing zero trust, in the following ways:\n\n\n- **Reduction of attack surface:** If credentials are compromised, the\nbreach will be limited to only the paths where the compromised account has\naccess.\n\n- **Protection against human error:** Users will not be able to perform\nactions that are not required for their role.\n\n- **Adherence to compliance:** Separation of duties and least privilege best\npractices are required for several compliance mandates such as SOC2 and\nHIPAA.\n\n- **Reduced system downtime:** By preventing everyone from accessing\ncritical parts of the software development lifecycle (SDLC), there is less\nlikelihood of downtime.\n\n\nGitLab provides a variety of different features that allow you to customize\nthe actions a user can perform which assist in the achievement of PoLP.\nThese features include:\n\n\n- **[Custom roles and granular security\npermissions](#custom-roles-and-granular-security-permissions):** Allows\ncreation of roles with permissions that are specific to particular functions\nrequired by the organization.\n\n- **[Security policies](#security-policies):** Allows policies to be created\nthat prevent insecure code from being merged into production branches\nwithout approval, and run security scanners regardless of your pipeline\ndefinition.\n\n- **[Branch protections and Code\nOwners](#branch-protections-and-code-owners):** Imposes further restrictions\non certain branches to control permissions such as who can merge, push, etc.\nto defined branches.\n\n- **[Compliance pipelines and\nframeworks](#compliance-pipelines-and-frameworks):** Identifies that your\nproject has certain compliance requirements or needs additional oversight,\nenforcing a pipeline configuration to the projects on which it is applied.\n\n\nIn this blog post, you'll learn each of the features mentioned, how they\nimprove your organization's security posture, as well as how to implement\nthem.\n\n\nWatch my video, which introduces you to achieving PoLP with GitLab:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/jvZ3eqWMeSY?si=DedSYiBNy2kTLJKo\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## Custom roles and granular security permissions\n\n\nGitLab allows you to create [custom\nroles](https://docs.gitlab.com/ee/user/custom_roles.html), which apply\nadditional permissions to base roles to meet the security needs of your\norganization. The available [base\nroles](https://docs.gitlab.com/ee/user/permissions.html#roles) are as\nfollows:\n\n\n- Guest\n\n- Reporter\n\n- Developer\n\n- Maintainer\n\n- Owner\n\n\nEach base role applies a particular set of permissions to a user. Base roles\napply different permissions for [group\nmembers](https://docs.gitlab.com/ee/user/permissions.html#group-members-permissions),\n[project\nmembers](https://docs.gitlab.com/ee/user/permissions.html#project-members-permissions),\nand in [project\nfeatures](https://docs.gitlab.com/ee/user/permissions.html#project-features-permissions).\nFor example, the table below shows which roles can view the project\n[dependency\nlist](https://docs.gitlab.com/ee/user/application_security/dependency_list/):\n\n\n| Base role    | Can view project dependency list     |\n\n| ---------- | ---------- |\n\n| Guest      | ❌       |\n\n| Reporter      | ❌       |\n\n| Developer      | ✅       |\n\n| Maintainer      | ✅       |\n\n| Owner       | ✅     |\n\n\n\u003Cbr>\u003C/br>\n\nThe dependency list also known as a software bill of materials\n([SBOM](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/)),\ndisplays your project's dependencies\n\nand key details about those dependencies. It makes sense that only those\nactively working on a project should\n\nbe able to see what dependencies are present to limit any exploitation of\nyour application using its dependencies.\n\n\nHowever, there are cases in which a Guest may need to see the SBOM to assist\nthe organization in\n\n[achieving\ncompliance](https://www.whitehouse.gov/briefing-room/presidential-actions/2021/05/12/executive-order-on-improving-the-nations-cybersecurity/).\nBy using custom roles, a new role can be created with all the limited\npermissions of the Guest role, and additionally, the ability to view the\nproject dependency list can be added. Therefore, we have a Guest assisting\nus with compliance with the least privileged access required for their job.\n\n\nWatch my video on custom roles and granular security permissions with\nGitLab:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/WyrhkpO5WkI?si=4B4mNYNK9UyNrru8\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n### Granular permissions\n\n\nAs of the GitLab 16.8 release, the following granular permissions can be\nadded to any base role:\n\n\n- Viewing project code\n\n- Viewing vulnerability reports\n\n- Changing the status of vulnerabilities\n\n- Viewing SBOMs\n\n- Approving merge requests\n\n- Managing project/group access tokens\n\n- Adding/removing group members\n\n- Archiving/unarchiving/removing projects\n\n- Admin Terraform state\n\n\nWe will continue to add [more granular\npermissions](https://docs.gitlab.com/ee/user/custom_roles/abilities.html)\nwith each GitLab release. You can learn more about our roadmap for this\nfeature by referring to the [Granular Security Permissions\nEpic](https://gitlab.com/groups/gitlab-org/-/epics/10684) and provide\nfeedback in the [customer feedback\nIssue](https://gitlab.com/gitlab-org/gitlab/-/issues/391760). You also have\nthe ability to contribute to GitLab and [develop your own granular\npermissions](https://docs.gitlab.com/ee/development/permissions/custom_roles.html).\n\n\n### Implementation prerequisites\n\nThe requirements for implementing custom roles are as follows: \n\n- Owner role in the top-level group in which you are creating the custom\nrole\n\n- Administrator for the self-managed instance in which you are creating the\ncustom role\n\n- GitLab Ultimate tier in the top-level group\n\n- A [personal access token with the API\nscope](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)\n\n\nTo see custom roles in action requires:\n\n- a private project within the top-level group or its subgroups\n\n- a guest user within the private project\n\n\nWhen you enable a custom role for a user with the Guest role, that user has\naccess to elevated permissions, and therefore:\n\n- is considered a billable user on self-managed GitLab\n\n- uses a seat on GitLab.com\n\n\n### Creating the custom role with granular permissions\n\n\nNow that you know the benefits of implementing custom roles with granular\npermissions, let's implement them within our GitLab instance:\n\n\n1. On the left sidebar, select **Search or go to**.\n    - In GitLab SaaS find and select the top-level group in which you want to create a custom role.\n    - In GitLab Self-Managed find and select **Admin Area**.\n2. Select **Settings > Roles and Permissions**.\n    - In GitLab Self-Managed use the top dropdown list to find and select the top-level group in which you want to create a custom role.\n3. Select **Add new role**.\n\n4. Under Base role to use as a template, select **Guest** for this tutorial.\n\n5. Under Role name, enter the custom role’s title.\n\n6. Under Permissions for the custom role, select **Read Vulnerability** for\nthis tutorial.\n\n7. Select **Create a new role**.\n\n\n![Create new role\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099455072.png)\n\n\n\u003Ccenter>\u003Ci>Interface for creating a custom role\u003C/i>\u003C/center>\n\n\u003Cp>\u003C/p>\n\n\nAfter creating the role you should be able to see the new custom role along\nwith its ID, Base role, and Permissions. Be sure to save the ID as it will\nbe used when we assign the custom role to a guest user.\n\n\n![Custom role\nscreen](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099455073.png)\n\n\n\u003Ccenter>\u003Ci>Security Auditor role created\u003C/i>\u003C/center>\n\n\u003Cp>\u003C/p>\n\n\nNow we must assign the custom role to a group or project member. This can be\ndone as follows:\n\n1. Invite a user as a direct member with the Guest role to your top-level\ngroup where the custom role was created.\n\n2. You can invite them to a sub-group or private project within the\ntop-level group as well.\n\n* The guest user should not be able to see any code within the project they\nhave been assigned to.\n\n* Open your terminal.\n\n3. Export the required environment variables:\n\n* Your [personal access token with API\nscope](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)\n\n\n```\n\n$ export TOKEN=glpat-XXXXXXXXXXXX\n\n$ echo $TOKEN\n\nglpat-XXXXXXXXXXXX\n\n\n```\n\n\n* The ID of the user we will be granting a custom role to. You can obtain\nthe user id by providing the username to the [User\nAPI](https://docs.gitlab.com/ee/api/users.html#list-users). For more\ninformation on using the GitLab API, see the [REST API\ndocumentation](https://docs.gitlab.com/ee/api/rest/).\n\n\n```\n\n$ curl \"https://gitlab.example.com/api/v4/users?username=fjdiaz\"\n\n[{\"id\":4710074,\"username\":\"fjdiaz\",\"name\":\"Fern\",\"state\":\"active\",\"locked\":false,\"avatar_url\":\"https://gitlab.com/uploads/-/system/user/avatar/4710074/avatar.png\",\"web_url\":\"https://gitlab.com/fjdiaz\"}]\n\n\n$ export USER_ID=4710074\n\n$ echo $USER_ID\n\n4710074\n\n```\n\n\n* The ID of the custom role. You can obtain the custom role ID from the ID\ncolumn in the [custom roles\nUI](https://docs.gitlab.com/ee/user/custom_roles.html#gitlab-saas) or the\n[member roles\nAPI](https://docs.gitlab.com/ee/api/member_roles.html#add-a-member-role-to-a-group).\n\n\n```\n\n$ export CUSTOM_ROLE_ID=1000782\n\n$ echo $CUSTOM_ROLE_ID\n\n1000782\n\n```\n\n\n* The ID of your group or project. You can obtain the group id from the\n[group UI](https://docs.gitlab.com/ee/user/group/#get-the-group-id) or using\nthe [groups API](https://docs.gitlab.com/ee/api/groups.html). You can obtain\nthe project ID from the [project\nUI](https://docs.gitlab.com/ee/user/project/working_with_projects.html#access-the-project-overview-page-by-using-the-project-id)\nor using the [projects API](https://docs.gitlab.com/ee/api/projects.html).\n\n\n```\n\n$ export GROUP_ID=10087220\n\n$ echo $GROUP_ID\n\n10087220\n\n\n$ export PROJECT_ID=45738177\n\n$ echo $PROJECT_ID\n\n45738177\n\n```\n\n\n4. Associate the guest user with the custom role using the appropriate\n[group or project\nAPIs](https://docs.gitlab.com/ee/api/members.html#edit-a-member-of-a-group-or-project).\n\n\n* If the user just needs to role in a project, update the project\nmembership:\n\n\n```\n\n\"Authorization: Bearer $TOKEN\" --data '{\"member_role_id\": $CUSTOM_ROLE_ID,\n\"access_level\": 10}'\n\"https://gitlab.example.com/api/v4/projects/$PROJECT_ID/members/$USER_ID\"\n\n```\n\n\n* If the user just needs to role in a group, update the group membership:\n\n\n```\n\n$ curl --request PUT --header \"Content-Type: application/json\" --header\n\"Authorization: Bearer $TOKEN\" --data '{\"member_role_id\": $CUSTOM_ROLE_ID,\n\"access_level\": 10}'\n\"https://gitlab.example.com/api/v4/groups/$GROUP_ID/members/$USER_ID\"\n\n```\n\n\nNow that the custom role has been applied to a guest user, when they login,\nthey can see the Vulnerability dashboard present in the Secure tab. Notice,\nhowever, that they are still not allowed to see the source code. \n\n\nThis is useful because it allows users to audit the system without being\nable to make changes to the code base, which applies the PoLP for those\nauditing the system for vulnerabilities.\n\n\n## Security policies\n\nGitLab provides [security\npolicies](https://docs.gitlab.com/ee/user/application_security/policies/) to\nhelp you achieve least privilege access. There are two different types of\nsecurity policies provided by GitLab:\n\n- [Scan Execution\npolicies](https://docs.gitlab.com/ee/user/application_security/policies/scan-execution-policies.html)\nallow project maintainers and administrators the confidence of knowing that\nthe scans they set up have not been changed, altered, or disabled.\n\n- [Merge Request Approval\npolicies](https://docs.gitlab.com/ee/user/application_security/policies/scan-result-policies.html)\nprevent insecure code from being merged into production without appropriate\napproval.\n\n\nSome examples of how both policy types can be used in unison to provide\nleast privilege access are as follows:\n\n- remove the ability for developers to disable security scanners\n\n- remove the ability for developers to merge insecure code\n\n\nPolicies are stored in a separate repo from the project they are being\napplied to called the Security Policy Project (SPP). This allows for\nseparate permissions to be set to the SPP vs. the application repo, thus\nstrengthening your ability to separate duties and apply PoLP.\n\n\n![Security policy\nhierarchy](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image19_aHR0cHM6_1750099455074.png)\n\n\n\u003Ccenter>\u003Ci>Security policy hierarchy\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nTo enforce the policies contained in an SPP you link it to a project,\nsubgroup, group, or multiples of each. An SPP can contain multiple policies\nbut they are enforced together. An SPP enforced on a group or subgroup\napplies to everything below the hierarchy, including all subgroups and their\nprojects.\n\n\nSecurity policies can be managed via the policy management UI as well as via\nyaml. Using the policy editor you can create, edit, and delete policies.\n\n\n![Policy management\ninterface](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image20_aHR0cHM6_1750099455076.png)\n\n\n\u003Ccenter>\u003Ci>Policy management interface\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nFeel free to leverage the [Simple Notes demo\nenvironment](https://gitlab.com/gitlab-de/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\nto try this yourself by following the provided [DevSecOps\ntutorial](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/).\n\n\n### Creating a Scan Execution policy\n\nNow let's take a look at how to create a Scan Execution policy. Before\ngetting started make sure you have met the following criteria:\n\n- GitLab Ultimate tier in the top-level group\n\n- Owner role to create/assign an SPP\n\n- Developer role or greater to create/edit/delete individual security\npolicies\n\n\nWe will be creating a policy that automatically runs a SAST scan with each\npipeline, regardless of the SAST template is defined within the\ngitlab-ci.yml:\n\n\n1. On the left sidebar, select **Search or go to** and search for the\nproject to which you wish to add a policy.\n\n2. On the project left sidebar, go to **Secure > Policies**.\n\n3. Select **New policy**.\n\n4.  In the **Scan Execution Policy** section, select **Select policy**.\n\n5. Complete the fields:\n    - **Name:** The name of the policy\n    - **Description:** The description of the Policy\n    - **Policy status:** Whether it is enabled or not\n    - **Actions:** What actions to take when the defined conditions are met\n\n![Scan Execution policy\nactions](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image15_aHR0cHM6_1750099455077.png)\n\n \u003Ccenter>\u003Ci>Scan Execution policy actions\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n  - **Conditions:** Conditions which must be met (a pipeline is triggered or on a set schedule) in order for an action to take place.\n\n    ![Scan Execution policy conditions](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750099455078.png)\n \u003Ccenter>\u003Ci>Scan Execution policy conditions\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n- Press the **Configure with a merge request** button.\n\n\nNow that the policy has been created, all we need to do is run a pipeline to\nsee that SAST will be present even if it is not defined in the\n.gitlab-ci.yml.\n\n\n### Creating a Merge Request Approval policy\n\n\nNow let's take a look at how to create a Merge Request Approval policy.\nBefore getting started make sure you have met the following criteria:\n\n- GitLab Ultimate tier in the top-level group\n\n- Owner role to create/assign an SPP\n\n- Developer role or greater to create/edit/delete individual security\npolicies\n\n- Security scanners added to project\n\n\nWe will be creating a policy that requires approval from project maintainers\nif any security scanner detects a vulnerability when compared with any\nbranch:\n\n\n1. On the left sidebar, select **Search or go to** and search for the\nproject to which you wish to add a policy.\n\n2. On the project left sidebar, go to **Secure > Policies**\n\n3. Select **New policy**\n\n4. In the **Merge Request Approval policy** section, select **Select\npolicy**.\n\n5. Complete the fields:\n    - **Name:** The name of the policy\n    - **Description:** The description of the policy\n    - **Policy status:** Whether it is enabled or not\n    - **Rules:** The conditions which must be met for an action (require approval) to take place.\n\n![Merge Request Approval policy\nrules](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image17_aHR0cHM6_1750099455079.png)\n\n\n\u003Ccenter>\u003Ci>Merge Request Approval policy rules\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n   - **Actions:** The action to be taken whenever the conditions in the rules (defined vulnerabilities/licenses detected) are met.\n\n![Merge Request Approval  policy\nactions](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099455080.png)\n\n\u003Ccenter>\u003Ci>Merge Request Approval  policy actions\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n   - **Override project approval settings:** If selected, the following choices will overwrite project settings but only affect the branches selected in the policy.\n\n![Merge Request Approval policy approval\nsettings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image21_aHR0cHM6_1750099455081.png)\n\n\n\u003Ccenter>\u003Ci>Merge Request Approval policy approval\nsettings\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\n6. Press the **Configure with a merge request** button.\n\n\nNow that the policy has been created, all we need to do is run a pipeline\nand if SAST detects any vulnerabilities then approvals will be required from\nthe selected approver before the code change can be merged. Merge Request\nApproval policies can be used with all GitLab security scanners, including\nlicense scanning.\n\n\n![Merge Request Approval policies blocking code from being merged in an\nMR](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099455082.png)\n\n\n\u003Ccenter>\u003Ci>Merge Request Approval policies blocking code from being merged\nin an MR\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\n## Branch protections and Code Owners\n\n[Branch\nprotections](https://docs.gitlab.com/ee/user/project/protected_branches.html)\nallow you to impose additional restrictions on particular branches within\nyour repository. This further strengthens the PoLP for the interactions on a\nparticular set of branches. \n\n\nFor example, a protected branch can control:\n\n- which users can merge into the branch\n\n- which users can push to the branch\n\n- if users can force push to the branch\n\n- if changes to files listed in the CODEOWNERS file can be pushed directly\nto the branch\n\n- which users can unprotect the branch\n\n\n### Applying branch protections\n\n\nBranch protections are available in all tiers and offerings of GitLab.\nBranch protections can be applied to a single project or a group of\nprojects. You can apply branch protections for required roles to push and\nmerge as follows:\n\n\n1. On the left sidebar, select **Search or go to** and find your project or\ngroup.\n\n2. Select **Settings > Repository**.\n\n3. Expand **Protected branches**.\n\n4. Select **Add protected branch**.\n    - For groups, from the **Branch** text box, type the branch name or a wildcard.\n    - For projects, from the **Branch** dropdown list, select the branch you want to protect.\n5. From the **Allowed to merge** list, select a role that can merge into\nthis branch.\n\n6. From the **Allowed to push and merge** list, select a role that can push\nto this branch.\n\n7. Select **Protect**.\n\n\nYou should now see the protected branch added to the list.\n\n\n![Protected branches\nsettings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image14_aHR0cHM6_1750099455082.png)\n\n\n\u003Ccenter>\u003Ci>Protected branches settings\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nThe Owner role is required to add branch protections to a group and the\nMaintainer role or greater is required to add branch protections to a\nproject.\n\n\n### Code Owners\n\nIf you want to further limit what files developers can perform changes on,\none of the best features to implement is [Code\nOwners](https://docs.gitlab.com/ee/user/project/codeowners/). Code Owners\nallows you to define who has the expertise for specific parts of your\nproject’s codebase. Defining the owners of files and directories in Code\nOwners will:\n\n\n- require owners to approve changes as well as merge requests before they\nmerge into a protected branch\n\n- identify owners by displaying the Code Owner names on the files and\ndirectories they own\n\n\nTo set up Code Owners, follow these steps:\n\n1. Create a CODEOWNERS file in your preferred location.\n\n2. Define some rules in the file following the Code Owners syntax reference.\nYou can configure all eligible approvers' approval rules and require Code\nOwner approval on a protected branch.\n\n3. Commit your changes, and push them up to GitLab.\n\n\nNow, when looking at files, you can see who the Code Owners are for a\nparticular file.\n\n\n![Code Owners displayed for\nfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099455083.png)\n\n\n\u003Ccenter>\u003Ci>Code Owners displayed for file\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nIf you implement Code Owner approvals, then when creating a merge request,\nthe Code Owners must approve before the code can be merged.\n\n\n![Code Owners\napprovals](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750099455084.png)\n\n\n\u003Ccenter>\u003Ci>Code Owners approvals\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\n### Additional approval settings\n\nThere are additional approval settings that can be applied before code can\nbe committed with a merge request. These additional approval settings are as\nfollows:\n\n- prevent approval by author\n\n- prevent approvals by users who add commits\n\n- prevent editing approval rules in merge requests\n\n- require user re-authentication (password or SAML) to approve\n\n\nAdditionally, whenever a commit is added, you can:\n\n- keep approvals\n\n- remove all approvals\n\n- remove approvals by Code Owners if their files changed\n\n\n![Additional Approval\nsettings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image12_aHR0cHM6_1750099455084.png)\n\n\n\u003Ccenter>\u003Ci>Additional Approval settings\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nTo configure additional approval settings you can perform the following\nsteps:\n\n1. On the left sidebar, select **Search or go to** and find your project.\n\n2. Select **Settings > Merge requests**.\n\n3. Scroll down to the **Merge request approvals** section.\n\n4. Under **Approval settings** select the approval settings you would like\nto apply.\n\n5. Press the **Save changes** button.\n\n\nThese can also be applied to your top-level group by performing the\nfollowing steps:\n\n1. On the left sidebar, select **Search or go to** and find your top-level\ngroup.\n\n2. Select **Settings > General**.\n\n3. Expand the **Merge request approvals** section.\n\n4. Under **Approval settings** select the approval settings you would like\nto apply.\n\n5. Press the **Save changes** button.\n\n\nBy leveraging these approval settings you can make sure that code always\nobtains oversight by a person who was not involved in creating the code,\nthereby preventing a conflict of interest.\n\n\n## Compliance pipelines and frameworks\n\nYou can create a compliance framework that is a label to identify that your\nproject has certain compliance requirements or needs additional oversight.\nThe label can optionally enforce compliance pipeline configuration to the\nprojects on which it is applied.\n\n\nFeel free to leverage the [Compliance Frameworks\nDemo](https://gitlab.com/gitlab-de/tutorials/security-and-governance/compliance-frameworks)\ngroup to see an example of compliance frameworks and their usage.\n\n\n### Create a compliance pipeline\n\nTo create a compliance pipeline, all you need to do is create a new project\nwhich will store a `.gitlab-ci.yml` file that we wish to use in another\nproject. The new compliance pipeline project can have separate permissions\nfrom the project to which you will apply it. This is beneficial because it\nprevents developers from making changes to pipelines that must run.\n\n\nYou can see I have created the following [pipeline\ndefinition](https://gitlab.com/gitlab-de/tutorials/security-and-governance/compliance-frameworks)\nwhich:\n\n- runs the SAST security scanner\n\n- runs the secret detection scanner\n\n- runs a SOC2 compliance job\n\n- runs the original pipeline defined in the project to which we will apply\nthis pipeline. This allows developers to focus on the actual application\ndevelopment and the compliance team to focus on defining the SOC2 rules.\n\n\n### Create and apply a compliance framework\n\nNow that the compliance pipeline for SOC2 has been defined, we must define a\ncompliance framework and apply it to our project. In this case, I will apply\nit to my Accounting Department project.\n\n\nTo create a compliance framework label, follow these steps:\n\n1. On the left sidebar, select **Search or go to** and find your group.\n\n2. Select **Settings > General**.\n\n3. Expand the **Compliance frameworks** section.\n\n4. Click the **Add framework** button.\n\n5. Create a new compliance framework and populate the following sections:\n    - **Name:** The name of your compliance framework\n    - **Description:** A description of your compliance framework\n    - **Compliance pipeline configuration:** The location of the compliance pipeline to run. \n    - **Background color:** A color for the compliance framework label\n\n![PoLP - image\n15](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750099455085.png)\n\n   \u003Ccenter>\u003Ci>Creating a compliance framework\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n6. Press the **Add framework** button.\n\n\nAnd now you should see your newly added framework under active compliance\nframeworks.\n\n\n![Active compliance\nframeworks](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750099455085.png)\n\n\n\u003Ccenter>\u003Ci>Active compliance frameworks\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nNow let’s go ahead and assign this compliance label to our Accounting\nDepartment project:\n\n\n1. On the left sidebar, select **Search or go to** and find your project.\n\n2. Select **Settings > General**.\n\n3. Expand **Compliance frameworks**.\n\n4. Select the compliance framework created above.\n\n\n![Adding a compliance\nframework](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099455086.png)\n\n\n\u003Ccenter>\u003Ci>Adding a compliance framework\u003C/i>\u003C/center>\n\n\n5. Select **Save changes**.\n\n\nThe project should now have the compliance framework label applied. \n\n\n![Project running a compliance\npipeline](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750099455086.png)\n\n\n\u003Ccenter>\u003Ci>Project running a compliance pipeline\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nThis enables separation of duties and prevents compliance pipelines from\nbeing altered by those without permissions.\n\n\nSecurity Policy Scope and Pipeline Execution\n\nOver the past several releases, GitLab has introduced two experimental\nfeatures, Security Policy Scope and Pipeline Execution, to make it even\neasier to adhere to PoLP. These features are very similar to Compliance\nPipelines and Compliance Frameworks and can be managed from GitLab’s\nsecurity policy UI.\n\n\n**Note:** These features are currently considered experimental. An\nexperiment is a feature that is in the process of being developed. It is not\nproduction ready. We encourage users to try experimental features and\nprovide feedback.\n\n\nThe [pipeline execution policy\naction](https://docs.gitlab.com/ee/user/application_security/policies/scan-execution-policies.html#pipeline-execution-policy-action)\nintroduces a new scan action type into Scan Execution policies for creating\nand enforcing custom CI in your target development projects. You can execute\na custom pipeline along with your current pipeline. This allows you to\nenforce compliance by always forcing particular actions to run that are not\njust security scanners and that cannot be overwritten by those without\npermissions.\n\n\n![Pipeline Execution policy scope\nselection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image18_aHR0cHM6_1750099455087.png)\n\n\u003Ccenter>\u003Ci>Pipeline Execution policy scope selection - insert code\nblock\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\n![Pipeline Execution policy scope\nselection](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099455/Blog/Content%20Images/Blog/Content%20Images/image13_aHR0cHM6_1750099455087.png)\n\n\u003Ccenter>\u003Ci>Pipeline Execution policy scope selection - link existing CI\nfile\u003C/i>\u003C/center>\u003Cp>\u003C/p>\n\n\nThe [Security policy\nscope](https://docs.gitlab.com/ee/user/application_security/policies/scan-result-policies.html#security-policy-scopes)\ncan be applied to either Merge Request Approval or Scan Execution policies.\nScopes enable you to administer policies with a particular scope, meaning\nyou can:\n\n\n- Include only projects containing a compliance framework label\n\n- Include or exclude selected projects from enforcement\n\n\nTo enable these experimental features, follow these steps:\n\n1. On the left sidebar, select **Search or go to** and find your top-level\ngroup.\n\n2. Select **Settings > General**.\n\n3. Expand **Permissions and group features**.\n\n4. Scroll down to the **Security policy management** section.\n\n5. Select the following checkboxes\n\n**Security policy pipeline execution action:** Create and enforce custom CI\njobs and scripts using this new policy action.\n\n6. **Security policy scopes:** Granularly scope each policy you create to\nprojects containing a compliance framework label, or a list of projects.\n\n7. **Enforce for all subgroups (optional):** Subgroups cannot change these\nsettings.\n\n8. Scroll down to the **Experiment and Beta features** section.\n\n9. Select the **Use Experiment and Beta features** checkbox.\n\n10. Scroll down and press the **Save changes** button.\n\n\nNow, whenever you are creating a security policy, the following options will\nbe available:\n\n\n- Inserting a CI code block (Scan Execution policy only)\n\n- Loading CI/CD code from file (Scan Execution policy only)\n\n- Linking an existing CI file from another project (Scan Execution policy\nonly)\n\n- Scoping a policy to projects with selected compliance framework (Group\nLevel only)\n\n- Scoping a policy towards specific projects (Group Level only)\n\n- Scoping a policy towards all projects in group (Group Level only)\n\n\nTo learn more about these features, check out the following documentation:\n\n- [Pipeline Execution Policy action (Scan Execution\npolicy)](https://docs.gitlab.com/ee/user/application_security/policies/scan-execution-policies.html#pipeline-execution-policy-action)\n\n- [Security Policy Scopes (Scan Execution\npolicy)](https://docs.gitlab.com/ee/user/application_security/policies/scan-execution-policies.html#security-policy-scopes)\n\n- [Security Policy Scopes (Merge Request Approval\npolicy)](https://docs.gitlab.com/ee/user/application_security/policies/scan-result-policies.html#security-policy-scopes)\n\n\n## Additional resources\n\n\nThanks for reading! These are some of the ways that GitLab allows you to\nstrengthen your organization's security posture through the enablement of\nPoLP. To learn more about GitLab and the other ways we can strengthen your\norganization's security throughout all parts of the SDLC, check out the\nfollowing links:\n\n\n- [GitLab Security and\nCompliance](https://about.gitlab.com/solutions/security-compliance/)\n\n- [GitLab Application Security\nDocumentation](https://docs.gitlab.com/ee/user/application_security/)\n\n- [GitLab DevSecOps Demo\nProject](https://gitlab.com/gitlab-de/tutorials/security-and-governance/devsecops/simply-vulnerable-notes)\n\n- [GitLab DevSecOps\nTutorial](https://gitlab-de.gitlab.io/tutorials/security-and-governance/devsecops/simply-vulnerable-notes/)\n\n- [GitLab Roles and Permissions\nDocumentation](https://docs.gitlab.com/ee/user/permissions.html)\n\n- [GitLab Custom Roles\nDocumentation](https://docs.gitlab.com/ee/user/custom_roles.html)\n\n- [GitLab Security Policies\nDocumentation](https://docs.gitlab.com/ee/user/application_security/policies/)\n\n- [GitLab Compliance Frameworks\nDocumentation](https://docs.gitlab.com/ee/user/group/compliance_frameworks.html)\n\n- [GitLab Code Owners\nDocumentation](https://docs.gitlab.com/ee/user/project/codeowners/)\n\n- [GitLab Branch Protections\nDocumentation](https://docs.gitlab.com/ee/user/project/protected_branches.html)\n",[3416,9,830,696],{"slug":4152,"featured":90,"template":700},"the-ultimate-guide-to-least-privilege-access-with-gitlab","content:en-us:blog:the-ultimate-guide-to-least-privilege-access-with-gitlab.yml","The Ultimate Guide To Least Privilege Access With Gitlab","en-us/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab.yml","en-us/blog/the-ultimate-guide-to-least-privilege-access-with-gitlab",{"_path":4158,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4159,"content":4165,"config":4171,"_id":4173,"_type":13,"title":4174,"_source":15,"_file":4175,"_stem":4176,"_extension":18},"/en-us/blog/the-ultimate-guide-to-token-management-at-gitlab",{"title":4160,"description":4161,"ogTitle":4160,"ogDescription":4161,"noIndex":6,"ogImage":4162,"ogUrl":4163,"ogSiteName":685,"ogType":686,"canonicalUrls":4163,"schema":4164},"The ultimate guide to token management at GitLab","Learn all the steps in the end-to-end process of identifying, managing, and securing tokens for improved security across the software development lifecycle.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097408/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_1097303277_6gTk7M1DNx0tFuovupVFB1_1750097407860.jpg","https://about.gitlab.com/blog/the-ultimate-guide-to-token-management-at-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"The ultimate guide to token management at GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Hakeem Abdul-Razak\"}],\n        \"datePublished\": \"2025-02-25\",\n      }",{"title":4160,"description":4161,"authors":4166,"heroImage":4162,"date":4168,"body":4169,"category":830,"tags":4170},[4167],"Hakeem Abdul-Razak","2025-02-25","Imagine this: You are an engineer at a growing tech company, and it’s 2 a.m. when you get an urgent call. A critical deployment pipeline has failed, and your team is scrambling to figure out why. After hours of digging, you realize someone revoked a personal access token belonging to an engineer who left the company a week ago. This token was tied to several key automation processes, and now your entire system is in chaos. How do you make sure it does not happen again?\n\nFollow this guide, which takes GitLab customers through the end-to-end process of identifying, managing, and securing their tokens. It is meant to be a handy supplement to the extensive [token overview documentation](https://docs.gitlab.com/ee/security/tokens) for GitLab administrators, developers, and security teams who need to ensure proper token management within their projects.\n\nHere's what is covered in this guide:\n- [How to select the right token for the job](#how-to-select-the-right-token-for-the-job)\n- [Token types](#token-types)\n- [Discovering your tokens](#discovering-your-tokens)\n    - [Credentials inventory](#credentials-inventory)\n- [Managing tokens in the GitLab UI and API](#managing-tokens-in-the-gitlab-ui-and-api)\n- [Token rotation and expiration management](#token-rotation-and-expiration-management)\n- [Token management best practices](#token-management-best-practices)\n    - [Service accounts](#service-accounts)\n\n## How to select the right token for the job\n\nChoosing the right token ensures optimal security and functionality based on your use case. \nTokens can be used for authenticating API requests, automating CI/CD pipelines, integrating third-party tools, managing deployments and repositories, and more.\n\n![Token management guide - flow chart for tokens](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097435/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097434869.png)\n\nFor the sake of simplicity, the chart illustrates a straightforward use case tied to single user ownership. For more information, check out our documentation of user roles and permissions at each [namespace](https://docs.gitlab.com/ee/user/permissions.html) (user/group) in your instance or top-level group. Example use cases could be as follows: \n\n- **Personal access tokens** ([PAT](https://docs.gitlab.com/user/profile/personal_access_tokens/#personal-access-token-scopes)) can be used by developers when a user's personal access and permissions are required. In this case, the credentials follow the status and permissions of the user, including the removal of access if the account loses access to a specific project or group (or is blocked entirely).   \n- **Project/group access tokens** ([PrAT](https://docs.gitlab.com/user/project/settings/project_access_tokens/#scopes-for-a-project-access-token)/[GrAT](https://docs.gitlab.com/user/group/settings/group_access_tokens/#scopes-for-a-group-access-token)) are recommended when access should be scoped to resources within a specific project/group, allowing anyone with a PrAT/GrAT to access those resources through mechanisms managed by assigned scopes.\n\n## Token types\n\nBelow is a list of GitLab tokens with their default prefixes and use cases. For more information, please visit the [GitLab Token overview page](https://docs.gitlab.com/ee/security/tokens/#available-scopes). \n\n| Tokens | Prefix  | Description |\n| :---: | :---: | :---: |\n| Personal access token | glpat | Access user-specific data |\n| OAuth 2.0 token |  gloas | Integrate with third-party applications using OAuth2.0 authentication protocol |\n| Impersonation token | glpat | Act on behalf of another user for administrative purposes |\n| Project access token | glpat | Access data from a specific project |\n| Group access token | glpat |  Access data from a specific group |\n| Deploy token | gldt |  Clone, push, and pull container registry images of a project without a user and a password |\n| Deploy keys | N/A | Allow read-only or read-write access to your repositories |\n| Runner authentication token | glrt | Authenticate GitLab Runners |\n| CI/CD job token  | glcbt | Automate CI/CD processes |\n| Trigger token | glptt | Trigger pipelines manually or programmatically |\n| Feed token | glft | Authenticate access to package/RSS feeds |\n| Incoming mail token  | glimt | Process incoming emails |\n| GitLab agent for Kubernetes token | glagent | Manage Kubernetes clusters via the GitLab agent |\n| SCIM tokens | glsoat | Enable SCIM integrations for user provisioning |\n| Feature flags client token | glffct | Enable feature flags programmatically |\n| Webhook token | N/A | User set secret token to secure webhook payloads and ensure that the requests are from GitLab |\n\n## Discovering your tokens\n\n### Credentials inventory\n\nOn GitLab Ultimate, administrators (GitLab Self-Managed) and top-level group owners of an enterprise organization (GitLab.com as of Version 17.5) can monitor the credentials in their namespace.\n\nThis inventory tracks token details such as:\n\n* Token type  \n  * Available tokens on [GitLab.com](https://docs.gitlab.com/ee/user/group/credentials_inventory.html)  \n  * Available tokens on [GitLab Self-Managed](https://docs.gitlab.com/ee/administration/credentials_inventory.html)  \n* Associated user accounts  \n* Token scopes, and creation and expiration dates  \n* Token last used IP addresses (as of GitLab 17.10)  \n* Token filtration based on the above user-defined parameters  \n* Ability to revoke and rotate those tokens\n\nA well-maintained credentials inventory helps identify over-permissioned tokens, and gives insight into credentials that may need to be rotated, ensuring a secure and efficient workflow.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/A9ONfnwswd0?si=4VIEUgJaD4daj81b&start=105\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n#### Credentials inventory API\n\nAs a complement to the UI, there is [ongoing development](https://gitlab.com/groups/gitlab-org/-/epics/16343) to release a credentials inventory API through the new /group/:id/manage [endpoint](https://docs.gitlab.com/ee/api/members.html#list-all-members-of-a-group-or-project). The credentials accessible under this endpoint are limited to enterprise [users](https://docs.gitlab.com/ee/user/enterprise_user/), and can be accessed by the top-level group owner of an enterprise organization. An example of the future API call would be:\n\n```console\ncurl --header \"PRIVATE-TOKEN: \u003Cpat>\" \"https://verified_domain.com/api/v4/groups/\u003Cgroup_id>/manage/personal_access_tokens\"           \n```\n### GitLab API\n\nThe GitLab API allows you to programmatically list and manage tokens within your organization. Key authentication-related endpoints support [various token types](https://docs.gitlab.com/ee/api/rest/authentication.html)), including personal, group, CI/CD tokens, and more. An example of using a personal access token to list all visible projects across GitLab for the authenticated user is:\n\n```console\ncurl --header \"PRIVATE-TOKEN: \u003Cyour_access_token>\" \\\n     \"https://gitlab.example.com/api/v4/projects\"\n\n```\n\nWatch this video to learn how to make API calls to the GitLab API.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/0LsMC3ZiXkA?si=vj871YH610jwQdFc\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Finding where tokens are used\n\nCustomers can find where tokens are used in different ways:\n* under **User Profile > [Access Tokens](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#view-the-time-at-and-ips-where-a-token-was-last-used)**\n* in credentials inventory\n* in audit events\n* via the API \n\nInformation on token usage is updated every 10 minutes for **last_used** and every minute for **last_used_ip**. \n\nThe ability to view IP addresses was introduced in GitLab 17.9, and is controlled by the **:pat_ip** feature flag. Follow these [steps to view the last time a token was used](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#view-the-time-at-and-ips-where-a-token-was-last-used), along with its last five distinct IP addresses.\n\n![Token management guide - personal access tokens settings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097435/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097434870.png)\n\n## Managing tokens in the GitLab UI and API\nThe following table includes videos detailing a few token creations in the UI and demonstrates their usage via the API.\n\n| Tokens     | GitLab UI    | GitLab API    |\n| ---------- | ---------- | ---------- |\n| Personal access token | [Documentation](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) and [video](https://youtu.be/v5Nj3Jy4vaI?t=3)  | [Documentation](https://docs.gitlab.com/ee/api/personal_access_tokens.html) and [video](https://youtu.be/v5Nj3Jy4vaI?t=43)  |\n| Group access token | [Documentation](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html#group-access-tokens) and [video](https://youtu.be/v5Nj3Jy4vaI?t=120)  | [Documentation](https://docs.gitlab.com/ee/api/group_access_tokens.html) and [video](https://youtu.be/v5Nj3Jy4vaI?t=157)  |\n| Project access token | [Documentation](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#project-access-tokens) and [video](https://youtu.be/v5Nj3Jy4vaI?t=254)  | [Documentation](https://docs.gitlab.com/ee/api/project_access_tokens.html) and [video](https://youtu.be/v5Nj3Jy4vaI?t=285)  |\n\n## Token rotation and expiration management\n\nImplementing token rotation and strict expiration policies reduces the risk of compromise and ensures compliance with security standards. Regular rotation and enforced expirations prevent stale credentials from becoming security vulnerabilities.\n\nPreviously, expired group and project access tokens were automatically deleted upon expiration, which made auditing and security reviews more challenging due to the lack of a record of inactive tokens. To address this, a [recent feature](https://gitlab.com/gitlab-org/gitlab/-/issues/462217) introduced the retention of inactive group and project access token records in the UI for 30 days after they became inactive. This enhancement aims to allow teams to track token usage, expiration, and revocation for better compliance and monitoring.\n\nTo be more proactive in your token rotation and expiration management, do the following: \n\n* Actively rotate your tokens via the UI or API. If you use the latter, be mindful of the [automatic token reuse detection](https://docs.gitlab.com/ee/api/personal_access_tokens.html#automatic-reuse-detection) security mechanism.  \n* Set an instance-wide [maximum lifetime limit](https://docs.gitlab.com/ee/administration/settings/account_and_limit_settings.html#limit-the-lifetime-of-access-tokens) for access tokens. \n\n### Token rotation API\n\nUntil GitLab 17.7, customers had to programmatically rotate access tokens with the API. Its counterpart is now available on the UI. Check out the video in the table below or follow the [documentation](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#use-the-ui) for guidance.\n\n### Token rotation snippets\n\nThe following table includes videos detailing the rotation of GitLab tokens. \n\n| Tokens | Prerequisites | GitLab UI | GitLab API |\n| :---: | :---: | ----- | ----- |\n| Personal access token | Scope: api\u000b | [Documentation](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) and [video](https://youtu.be/v5Nj3Jy4vaI?t=76)  | [Documentation](https://docs.gitlab.com/ee/api/personal_access_tokens.html#rotate-a-personal-access-token) and [video](https://youtu.be/v5Nj3Jy4vaI?t=92)  |\n| Group access token | Scope: api and Role(s): owner | [Documentation](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html#create-a-group-access-token-using-ui) and [video](https://youtu.be/v5Nj3Jy4vaI?t=203)  | [Documentation](https://docs.gitlab.com/ee/api/group_access_tokens.html) and [video](https://youtu.be/v5Nj3Jy4vaI?t=214)  |\n| Project access token | Scope: api and Role(s): owner, maintainer | [Documentation](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#create-a-project-access-token) and [video](https://youtu.be/v5Nj3Jy4vaI?t=335)  | [Documentation](https://docs.gitlab.com/ee/api/project_access_tokens.html) and [video](https://youtu.be/v5Nj3Jy4vaI?t=349)  |\n\n## Token management best practices\n\n### Principle of least privilege\n\nMitigate risk by restricting assigned permissions to tokens required for their respective tasks. This allows you to proactively predict and troubleshoot points of failure in your systems. You can do this by: \n\n* Selecting the right token for the right job. See the flowchart.  \n* Assign only the required scopes when creating a token. For example, use read-only scopes for tokens with auditor-like jobs. See [roles](https://docs.gitlab.com/ee/user/permissions.html#roles).  \n* Avoid granting administrative privileges unless specifically required.  \n* Enforce instance-wide default token [lifetimes](https://docs.gitlab.com/ee/administration/settings/account_and_limit_settings.html#set-a-lifetime-1).  \n* Regularly review and audit token permissions to ensure they align with current operational needs.  \n* Revoke tokens once the task is complete.\n\n### Service accounts\n\n[Service accounts](https://docs.gitlab.com/ee/user/profile/service_accounts.html) ensure tokens are tied to non-human entities, separating them from individual user accounts and reducing dependency on specific users. Instead of using personal accounts to generate tokens for automation, create service accounts with limited scopes. Benefits include:\n\n* Usage of service account tokens in CI/CD pipelines to avoid disruptions caused by user account changes  \n* Programmatically automate rotation processes, as personal accounts remain unaffected  \n* Clearer monitoring and auditing trail of actions taken by service accounts  \n* Service accounts with [no expiration](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-service-account-personal-access-token-with-no-expiry-date) date  \n* Does not consume [a license seat](https://docs.gitlab.com/user/profile/service_accounts/#create-a-service-account)\n\nGitLab plans to release a new [Service Accounts UI](https://gitlab.com/groups/gitlab-org/-/epics/9965) as a counterpart to its [API-based creation](https://docs.gitlab.com/ee/api/user_service_accounts.html#create-a-service-account-user), designed to simplify the management of service accounts and their associated tokens. Check out the demo below on the programmatic usage of service accounts.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/oZvjg0SCsqY?si=cj-0LjfeonLGXv9u\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Vulnerability tools\n\nLeverage GitLab’s built-in security tools to identify and mitigate vulnerabilities associated with token usage. For maximum coverage, it is recommended to use them all in tandem.\n\n* [Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/): Scans your repository for hardcoded secrets like API tokens, passwords, and other sensitive information. View the [list of detected secrets](https://docs.gitlab.com/ee/user/application_security/secret_detection/detected_secrets.html).  \n* [Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/): Analyzes your source code for security vulnerabilities and [provides reports with UI findings in merge requests](https://docs.gitlab.com/ee/user/application_security/sast/#features), among other features.  \n* [Dependency Scanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/): Ensures that third-party libraries used in your project do not expose token-related vulnerabilities. \n\n### Audit logs and monitoring\n\nMaintain token health by regularly reviewing audit logs and token usage, instance- and/or group-wide.\n\n* [Audit events](https://docs.gitlab.com/ee/user/compliance/audit_events.html): Enable audit event logging in GitLab to track token-related activities such as creation, usage, deletion and unusual API calls (unpermitted parameters in logs, and consistent triggers of the rate limiter). \n* [IP allowlisting](https://docs.gitlab.com/ee/administration/reporting/ip_addr_restrictions.html#configure-ip-address-restrictions): Helps prevent malicious users from hiding their activities behind multiple IP addresses.  \n* [Alerts](https://docs.gitlab.com/ee/operations/incident_management/alerts.html): Set up alerts for unusual activities (trigger paging for on-call rotations or be used to create incidents).  \n* [Credentials inventory](https://docs.gitlab.com/ee/administration/credentials_inventory.html): Complete control of all available access tokens with the ability to revoke as needed.  \n* [Notifications](https://docs.gitlab.com/ee/user/profile/notifications.html): Proactively handle any token (group, project, and personal) expiration notification emails you receive. Based on customer demand, this feature was recently extended to include 30-day and 60-day notifications from the seven-day default.   \n* [Webhooks](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#create-a-webhook): Access token webhooks can be configured on groups and projects to send seven-day token expiry events. This feature was also recently extended to include 30-day and 60-day notifications behind the **:extended_expiry_webhook_execution_setting** feature flag (disabled by default).\n\n## What's next\n\nWith GitLab’s large token catalog, there are ongoing [plans](https://gitlab.com/gitlab-org/gitlab/-/issues/502630) for consolidation with a focus on the lifetime, fine-grained scopes, consistent management, and usage. Our current prioritized token-related features include a complete UI for service accounts, additional credential types in the credentials inventory, and improved auditing for tokens and service accounts.\n\n> Sign up for a [free trial of GitLab Ultimate](https://about.gitlab.com/free-trial/) to start using token management.",[9,830,495,696,694],{"slug":4172,"featured":90,"template":700},"the-ultimate-guide-to-token-management-at-gitlab","content:en-us:blog:the-ultimate-guide-to-token-management-at-gitlab.yml","The Ultimate Guide To Token Management At Gitlab","en-us/blog/the-ultimate-guide-to-token-management-at-gitlab.yml","en-us/blog/the-ultimate-guide-to-token-management-at-gitlab",{"_path":4178,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4179,"content":4184,"config":4190,"_id":4192,"_type":13,"title":4193,"_source":15,"_file":4194,"_stem":4195,"_extension":18},"/en-us/blog/there-is-no-mlops-without-devsecops",{"title":4180,"description":4181,"ogTitle":4180,"ogDescription":4181,"noIndex":6,"ogImage":1276,"ogUrl":4182,"ogSiteName":685,"ogType":686,"canonicalUrls":4182,"schema":4183},"Building GitLab with GitLab: Why there is no MLOps without DevSecOps","Follow along as data scientists adopt DevSecOps practices and enjoy the benefits of automation, repeatable workflows, standardization, and automatic provisioning of infrastructure.","https://about.gitlab.com/blog/there-is-no-mlops-without-devsecops","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Building GitLab with GitLab: Why there is no MLOps without DevSecOps\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"William Arias\"}],\n        \"datePublished\": \"2023-10-05\",\n      }",{"title":4180,"description":4181,"authors":4185,"heroImage":1276,"date":4187,"body":4188,"category":718,"tags":4189},[4186],"William Arias","2023-10-05","Building predictive models requires a good amount of experimentation and\niterations. Data scientists building those models usually implement\nworkflows involving several steps such as data loading, processing,\ntraining, testing, and deployment. Such workflows or data science pipelines\ncome with a set of challenges on their own; some of these common challenges\nare:\n\n- prone to error due to manual steps\n\n- experimentation results that are hard to replicate\n\n- long training time of machine learning (ML) models \n\n\nWhen there is a challenge, there is also an opportunity; in this case, those\nchallenges represent an opportunity for data scientists to adopt DevSecOps\npractices and enjoy the benefits of automation, repeatable workflows,\nstandardization, and automatic provisioning of infrastructure needed for\ndata-driven applications at scale.\n\n\nThe [Data Science team at\nGitLab](https://about.gitlab.com/handbook/business-technology/data-team/organization/data-science/)\nis now utilizing the GitLab DevSecOps Platform in their workflows,\nspecifically to:\n\n- enhance experiment reproducibility by ensuring code and data execute in a\nstandardized container image\n\n- automate training and re-training of ML models with GPU-enabled CI/CD\n\n- leverage ML experiment tracking, storing the most relevant metadata and\nartifacts produced by data science pipelines automated with CI\n\n\nAt GitLab, we are proponents of \"dogfooding\" our platform and sharing how we\nuse GitLab to build GitLab. What follows is a detailed look at the Data\nScience team's experience.\n\n\n### Enhancing experiment reproducibility \n\nA baseline step to enhance reproducibility is having a common and standard\nexperiment environment for all data scientists to run experiments in their\nJupyter Notebooks. A standard data science environment ensures that all team\nmembers use the same software dependencies. A way to achieve this is by\nbuilding a container image with all the respective dependencies under\nversion control and re-pulling it every time a new version of the code is\nrun. This process is illustrated in the figure below:\n\n\n![build](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/build-2.png)\n\nData science image of automatic build using GitLab CI \n\n{: .note.text-center}\n\n\nYou might wonder if the image gets built every time there is a new commit.\nThe answer is \"no\" since that would result in longer execution times, and\nthe image dependencies versions don’t change frequently, rendering it\nunnecessary to build it every time there is a new commit. Therefore, once\nthe standard image is automatically built by the pipeline, it is pushed to\nthe GitLab Container Registry, where it is stored and ready to be pulled\nevery time changes to the model code are introduced, and re-training is\nnecessary.\n\n\n![registry](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/registry.png)\n\nGitLab Container Registry with image automatically built and pushed by a CI\npipeline\n\n{: .note.text-center}\n\n\nChanges to the image dependencies or Dockerfile require a [merge\nrequest](https://docs.gitlab.com/ee/user/project/merge_requests/) and an\napproval process.\n\n\n### How to build the data science image using GitLab CI/CD\n\nConsider this project structure:\n\n\n```\n\nnotebooks/\n\n.gitlab-ci.yml\n\nDockerfile\n\nconfig.yml\n\nrequirements.txt\n\n```\n\nGitLab's Data Science team already had a pre-configured JupyterLab image\nwith packages such as [gitlabds](https://pypi.org/project/gitlabds/1.0.0/)\nfor common data preparation tasks and modules to enable Snowflake\nconnectivity for loading raw data. All these dependencies are reflected in\nthe Dockerfile at the root of the project, plus all the steps necessary to\nbuild the image: \n\n\n```\n\nFROM nvcr.io/nvidia/cuda:12.1.1-base-ubuntu22.04\n\nCOPY .    /app/\n\nWORKDIR /app\n\nRUN apt-get update\n\nRUN apt-get install -y python3.9\n\nRUN apt-get install -y python3-pip\n\nRUN pip install -r requirements.txt\n\n```\n\n\nThe instructions to build the data science image start with using Ubuntu\nwith CUDA drivers as a base image. We are using this baseline image because,\nmoving forward, we will use GPU hardware to train models. The rest of the\nsteps include installing Python 3.9 and the dependencies listed in\n`requirements.txt` with their respective versions. \n\n\nAutomatically building the data science image using [GitLab\nCI/CD](https://about.gitlab.com/topics/ci-cd/) requires us to create the\n`.gitlab-ci.yml ` at the root of the project and use it to describe the jobs\nwe want to automate. For the time being, let’s focus only on the\n`build-ds-image`job:\n\n\n```\n\nvariables:\n  DOCKER_HOST: tcp://docker:2375\n  MOUNT_POINT: \"/builds/$CI_PROJECT_PATH/mnt\"\n  CONTAINER_IMAGE: \"$CI_REGISTRY_IMAGE/main-image:latest\"\n\nstages:\n    - build\n    - train\n    - notify\ninclude:\n  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'\nworkflow:\n  rules:\n    - if: $CI_PIPELINE_SOURCE == \"merge_request_event\"\n    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS\n      when: never\n\nbuild-ds-image:\n  tags: [ saas-linux-large-amd64 ]\n  stage: build\n  services:\n    - docker:20.10.16-dind\n  image:\n    name: docker:20.10.16\n  script:\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n    - docker build -t $CONTAINER_IMAGE .\n    - docker push $CONTAINER_IMAGE\n  rules:\n    - if: '$CI_PIPELINE_SOURCE == \"merge_request_event\" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'\n      changes:\n       - Dockerfile\n       - requirements.txt\n\n  allow_failure: true\n```\n\n\nAt a high level, the job `build-ds-image`:\n\n- uses a docker-in-docker service (dind) necessary to create docker images\nin GitLab CI/CD.\n\n- uses [predefined variables](link) to log into the GitLab Container\nRegistry, build the image, tag it using $CONTAINER_IMAGE variable, and push\nit to the registry. These steps are declared in the script section lines.\n\n- leverages a  `rules` section to evaluate conditions to determine if the\njob should be created. In this case, this job runs only if there are changes\nto the Dockerfile and requirements.txt file and if those changes are created\nusing a merge request.\n\n\nThe conditions declared in `rules` helps us optimize the pipeline running\ntime since the image gets rebuilt only when necessary.\n\n\nA complete pipeline can be found in this example project, along with\ninstructions to trigger the automatic creation of the data science image:\n[Data Science CI\npipeline](https://gitlab.com/gitlab-data/data-science-ci-example/-/blob/main/.gitlab-ci.yml?ref_type=heads).\n\n\n### Automate training and re-training of ML models with GPU-enabled CI/CD\n\nGitLab offers the ability to leverage GPU hardware and, even better, to get\nthis hardware automatically provisioned to run jobs declared in the\n.gitlab-ci.yml file. We took advantage of this capability to train our ML\nmodels faster without spending time setting up or configuring graphics card\ndrivers. Using GPU hardware ([GitLab\nRunners](https://docs.gitlab.com/ee/ci/runners/saas/gpu_saas_runner.html))\nrequires us to add this line to the training job: \n\n\n```\n\ntags:\n        - saas-linux-medium-amd64-gpu-standard\n```\n\n\nThe tag above will ensure that a GPU GitLab Runner automatically picks up\nevery training job.\n\nLet’s take a look at the entire training job in the .gitlab-ci.yml file and\nbreak down what it does:\n\n\n```\n\ntrain-commit-activated:\n    stage: train\n    image: $CONTAINER_IMAGE\n    tags:\n        - saas-linux-medium-amd64-gpu-standard\n    script:\n        - echo \"GPU training activated by commit message\"\n        - echo \"message passed is $CI_COMMIT_MESSAGE\"\n        - notebookName=$(echo ${CI_COMMIT_MESSAGE/train})\n        - echo \"Notebook name $notebookName\"\n        - papermill -p is_local_development False -p tree_method 'gpu_hist' $notebookName -\n    rules:\n        - if: '$CI_COMMIT_BRANCH == \"staging\"'\n          when: never\n        - if: $CI_COMMIT_MESSAGE =~ /\\w+\\.ipynb/\n          when: always\n          allow_failure: true\n    artifacts:\n      paths:\n        - ./model_metrics.md\n````\n\n\nLet’s start with this block:\n\n\n```\n\ntrain-commit-activated:\n    stage: train\n    image: $CONTAINER_IMAGE\n    tags:\n        - saas-linux-medium-amd64-gpu-standard\n```\n\n\n- **train-commit-activated** This is the name of the job. Since the model\ntraining gets activated given a specific pattern in the commit message, we\nuse a descriptive name to easily identify it in the larger pipeline.\n\n- **stage: train** This specifies the pipeline stage where this job belongs.\nIn the first part of the CI/CD configuration, we defined three stages for\nthis pipeline: `build`, `train`,  and `notify`. This job comes after\nbuilding the data science container image. The order is essential since we\nfirst need the image built to run our training code in it.\n\n- **image: $CONTAINER_IMAGE** Here, we specify the Docker image built in the\nfirst job that contains the CUDA drivers and necessary Python dependencies\nto run this job. $CONTAINER_IMAGE is a user-defined variable specified in\nthe variables section of the .gitlab-ci.yml file. \n\n- **tags: saas-linux-medium-amd64-gpu-standard** As mentioned earlier, using\nthis line, we ask GitLab to automatically provision a GPU-enabled Runner to\nexecute this job.\n\n\nThe second block of the job:\n\n\n```\n\nscript:\n        - echo \"GPU training activated by commit message\"\n        - echo \"message passed is $CI_COMMIT_MESSAGE\"\n        - notebookName=$(echo ${CI_COMMIT_MESSAGE/train})\n        - echo \"Notebook name $notebookName\"\n        - papermill -p is_local_development False -p tree_method 'gpu_hist' $notebookName -\n```\n\n\n- **script** This section contains the commands in charge of running the\nmodel training. The execution of this job is conditioned to the contents of\nthe  commit message. The commit message must have the name of the Jupyter\nNotebook that contains the actual model training code.\n\n\nThe rationale behind this approach is that we wanted to keep the data\nscientist workflow as simple as possible. The team had already adopted the\n[modeling\ntemplates](https://gitlab.com/gitlab-data/data-science/-/tree/main/templates)\nto start building predictive models quickly. Plugging the CI pipeline into\ntheir modeling workflow was a priority to ensure productivity would remain\nintact. With these steps:\n\n\n```\n\nnotebookName=$(echo ${CI_COMMIT_MESSAGE/train})\n        - echo \"Notebook name $notebookName\"\n        - papermill -p is_local_development False -p tree_method 'gpu_hist' $notebookName -\n```\n\n\nThe CI pipeline captures the name of the Jupyter Notebook with the training\nmodeling template and passes parameters to ensure\n[XGBoost](https://xgboost.readthedocs.io/en/stable/) uses the provisioned\nGPU. You can find an example of the Jupyter modeling template that is\nexecuted in this job\n[here](https://gitlab.com/gitlab-data/data-science-ci-example/-/blob/main/notebooks/training_example.ipynb?ref_type=heads).\n\n\nOnce the data science image is built, it can be reutilized in further model\ntraining jobs. The `train-commit-activated` job pulls the image from the\nGitLab Container Registry and utilizes it to run the ML pipeline defined in\nthe training notebook. This is illustrated in the `CI Job - Train model` in\nthe figure below:\n\n\n![training](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/training_job.png)\n\nTraining job executes ML pipeline defined in the modeling notebook\n\n{: .note.text-center}\n\n\nSince our image contains CUDA drivers and GitLab automatically provisions\nGPU-enabled hardware, the training job runs significantly faster with\nrespect to standard hardware.\n\n\n### Using GitLab ML experiment tracker\n\nEach model training execution triggered using GitLab CI is an experiment\nthat needs tracking. Using Experiment tracking in GitLab helps us to record\nmetadata that comes in handy to compare model performance and collaborate\nwith other data scientists by making result experiments available for\neveryone and providing a detailed history of the model development.\n\n\n![experiments](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/experiments.png)\n\nExperiments automatically logged on every CI pipeline GPU training run \n\n{: .note.text-center}\n\n\nEach model artifact created can be traced back to the pipeline that\ngenerated it, along with its dependencies:\n\n\n![traceability](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/traceability_small.png)\n\nModel traceability from pipeline run to candidate details\n\n{: .note.text-center}\n\n\n### Putting it all together\n\nWhat is machine learning without data to learn from? We also leveraged the\n[Snowflake](https://www.snowflake.com/en/) connector in the model training\nnotebook and automated the data extraction whenever the respective commit\ntriggers a training job. Here is an architecture of the current solution\nwith all the parts described in this blog post:\n\n\n![process](https://about.gitlab.com/images/blogimages/2023-10-04-there-is-no-mlops-without-devsecops/training_fixed.png)\n\nData Science pipelines automated using GitLab DevSecops Platform\n\n{: .note.text-center}\n\n\n| Challenge | Solution |\n\n| ------ | ------ | \n\n|Prone to error due to manual steps | Automate steps with [GitLab\nCI/CD](https://docs.gitlab.com/ee/ci/) |\n\n|Experimentation results that are hard to replicate    |  Record metadata\nand model artifacts with [GitLab Experiment\nTracker](https://docs.gitlab.com/ee/user/project/ml/experiment_tracking/)   \n|\n\n|The long training time of machine learning models     |  Train models with\n[GitLab SaaS GPU\nRunners](https://docs.gitlab.com/ee/ci/runners/saas/gpu_saas_runner.html)  |\n\n\nIterating on these challenges is a first step towards MLOps, and we are at\nthe tip of the iceberg; in coming iterations, we will adopt security\nfeatures to ensure model provenance (software bill of materials) and code\nquality, and to monitor our ML workflow development with value stream\ndashboards. But so far, one thing is sure: **There is no MLOps without\nDevSecOps**.\n\n\nGet started automating your data science pipelines, follow this\n[tutorial](https://about.gitlab.com/handbook/business-technology/data-team/platform/ci-for-ds-pipelines/)\nand clone this\n[data-science-project](https://gitlab.com/gitlab-data/data-science-ci-example)\nto follow along and watch this demo of using GPU Runners to train\n[XGBoost](https://xgboost.readthedocs.io/en/stable/) model.\n\n\nSee how data scientists can train ML models with GitLab GPU-enabled Runners\n(XGBoost 5-minute demo):\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/tElegG4NCZ0?si=L1IZfx_UGv6u81Gk\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## More \"Building GitLab with GitLab\" blogs\n\nRead more of our \"Building GitLab with GitLab\" series:\n\n- [How we use Web API fuzz\ntesting](https://about.gitlab.com/blog/building-gitlab-with-gitlab-api-fuzzing-workflow/)\n\n- [How GitLab.com inspired GitLab\nDedicated](https://about.gitlab.com/blog/building-gitlab-with-gitlabcom-how-gitlab-inspired-dedicated/)\n",[9,851,495,720],{"slug":4191,"featured":6,"template":700},"there-is-no-mlops-without-devsecops","content:en-us:blog:there-is-no-mlops-without-devsecops.yml","There Is No Mlops Without Devsecops","en-us/blog/there-is-no-mlops-without-devsecops.yml","en-us/blog/there-is-no-mlops-without-devsecops",{"_path":4197,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4198,"content":4204,"config":4210,"_id":4212,"_type":13,"title":4213,"_source":15,"_file":4214,"_stem":4215,"_extension":18},"/en-us/blog/tips-to-configure-browser-based-dast-scans",{"title":4199,"description":4200,"ogTitle":4199,"ogDescription":4200,"noIndex":6,"ogImage":4201,"ogUrl":4202,"ogSiteName":685,"ogType":686,"canonicalUrls":4202,"schema":4203},"Tips to configure browser-based DAST scans","Learn how to use the browser-based analyzer with common dynamic application security testing settings, based on web application attributes, to ensure successful scans.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659561/Blog/Hero%20Images/securitycheck.png","https://about.gitlab.com/blog/tips-to-configure-browser-based-dast-scans","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tips to configure browser-based DAST scans\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Julie Byrne\"},{\"@type\":\"Person\",\"name\":\"Jerez Solis\"}],\n        \"datePublished\": \"2023-11-14\",\n      }",{"title":4199,"description":4200,"authors":4205,"heroImage":4201,"date":4207,"body":4208,"category":830,"tags":4209},[4206,1894],"Julie Byrne","2023-11-14","The GitLab Dynamic Application Security Testing\n([DAST](https://docs.gitlab.com/ee/user/application_security/dast/)) scan\nuses an actively running environment to crawl the application and find\nmisconfigurations of your application server or incorrect assumptions about\nsecurity controls that may not be visible from the source code. GitLab now\nprovides a proprietary [browser-based\nanalyzer](https://docs.gitlab.com/ee/user/application_security/dast/browser_based.html)\nfor scanning applications that make heavy use of JavaScript, including\nsingle-page web applications. The DAST scan needs to be configured properly\nto account for various web application attributes, including authentication\nmechanism, authenticated landing page, and page load times. In this\ntutorial, you will learn common configurations that have helped our\ncustomers use the browser-based analyzer to successfully implement DAST\nscans. \n\n\n## General considerations\n\n\nThe browser-based DAST scan takes the URL of the application it's supposed\nto scan from the `DAST_WEBSITE` environment variable. This should point to a\ntest environment - you should not run a DAST scan against a production\nenvironment, even if you are only running a passive scan. For ephemeral\nenvironments that are deployed as part of the CI/CD pipeline, you can save\nthe URL of the environment as an artifact `environment_url.txt`, which will\nthen be used by the [DAST scan template\njob](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml)\nto set the `DAST_WEBSITE` variable.  The [GitLab Auto DevOps deploy template\njob](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml)\nhas an example of this. \n\n\nDepending on the size of the web application, a DAST scan may take an hour\nor more to complete. You will want to ensure that whatever runner is used to\nperform the DAST scan has a [job timeout\nvalue](https://docs.gitlab.com/ee/ci/runners/configure_runners.html#set-maximum-job-timeout-for-a-runner)\nset to be long enough to allow the scan to complete. Similarly, you should\nensure that the [project level CI/CD\ntimeout](https://docs.gitlab.com/ee/ci/pipelines/settings.html#set-a-limit-for-how-long-jobs-can-run)\nis sufficient to allow the job to complete. **Note:** Shared runners on\ngitlab.com have a runner timeout of 180 minutes, regardless of the project\nCI/CD timeout set.\n\n\n## Configuration options for websites requiring authentication\n\n\nMany web applications require a user to log in to access the site. Logins\ncan be implemented as basic http authentication or, more commonly, as form\nauthentication.  For form authentication, the login form might be\nimplemented in one of several ways:\n\n\n1. username and password fields on the main website landing page\n\n2. a login button that opens a modal (also called a modal window or\nlightbox) that displays in front of the page and disables all other page\ncontent until the login is completed or the modal is closed; there is not a\nseparate URL associated with a modal window\n\n3. a login button that opens a new window, with its own URL \n\n\nAdditionally, the form may be either a single-step form, where the username\nand password fields are on the same page, or a multi-step form, where the\nusername and password fields are on separate pages.  \n\n\nWhen running a DAST scan, the analyzer must know how to\n[authenticate](https://docs.gitlab.com/ee/user/application_security/dast/authentication.html). \nWe need to specify these details via the appropriate variables.  \n\n\nThe `DAST_USERNAME` and `DAST_PASSWORD` variables specify the login\ncredentials to be used. The variable values should be set via masked\nvariables at the project level, not included within the `.gitlab-ci.yml`\nfile.\n\n\n### URL variable values\n\n\nVarious URL values must also be specified: \n   - `DAST_AUTH_URL` - the URL of the login page\n   - `DAST_WEBSITE` - specifies the URL of the page the user is redirected to after logging in\n\n**Note:** If your website uses authentication with a login button that opens\na new window with its own URL, you should specify the URL of that new window\nas the `DAST_AUTH_URL` value.  \n\n\nGitLab enhancements are currently being implemented to support cases where\nadditional actions must be taken post-login prior to being brought to the\nmain site. See [this\nepic](https://gitlab.com/groups/gitlab-org/-/epics/11585)  for details: DAST\nbrowser-based analyzer multi-step login form does not support 'keep me\nsigned in' workflow (AzureAD).\n\n\n### Field variable values\n\n\n`FIELD` variables specify the page elements used. These values can typically\nbe identified by inspecting the page source. For single-step login pages,\nyou will need to specify:\n - `DAST_USERNAME_FIELD`\n - `DAST_PASSWORD_FIELD`\n - `DAST_SUBMIT_FIELD` \n\nFor multi-step logins, you would instead specify:\n\n - `DAST_FIRST_SUBMIT_FIELD` - the button clicked after entering the username\n - `DAST_SUBMIT_FIELD` - the button clicked after entering the password\n\nIf your login button opens a modal, you should also specify\n`DAST_BROWSER_PATH_TO_LOGIN_FORM`, which provides the path of elements to\nclick to get from the initial login URL to the login fields.\n\n\n#### Examples\n\n\n![username email\nfield](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683113/Blog/Content%20Images/username_email_field.png)\n\n\nIn this example, you can target the input element for the username field in\ndifferent ways. Keep in mind that the selector you chose should be resilient\nto the application changing. We recommend to use the `id` and `name`\nattributes, as these are generally unique on a page and rarely change.\n\n\n```\n\nDAST_USERNAME_FIELD: \"id:user_login\"\n\nDAST_USERNAME_FIELD: \"name:user[login]\"\n\nDAST_USERNAME_FIELD: \"css:input[type=text]\"\n\n```\n\n\nThe same process can be followed for the password field. For example:\n\n\n```\n\nDAST_PASSWORD_FIELD: \"id:user_password\"\n\nDAST_PASSWORD_FIELD: \"name:user[password]\"\n\nDAST_PASSWORD_FIELD: \"css:input[type=password]\"\n\n```\n\n\n**Submit/Sign in/Login button**\n\n\n![login](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749683114/Blog/Content%20Images/login.png)\n\n\nYou can target the submit/sign in/login button using this selector:\n\n\n```\n\nDAST_SUBMIT_FIELD: \"css:button[type=submit]\"\n\n```\n\n\n**Note:** If the submit button is not a `\u003Cbutton>`, but an input element in\nthe form of `\u003Cinput type=\"submit\" name=\"login\">`, you can use one of the\nfollowing selectors:\n\n\n```\n\nDAST_SUBMIT_FIELD: \"css:input[type=submit]\"\n\nDAST_SUBMIT_FIELD: \"css:input[name=login]\"\n\n```\n\n\n### Other variables to set\n\n\nIf the username and password fields are on separate pages, DAST has to wait\nafter submitting the username before looking for the password field.\n\n\nThe `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` variable, with a default value\nof 800ms, specifies the wait time. This time can be increased if the login\nresponse time is slow.\n\n\nIf the website has a large JavaScript file that is required to load the\ntarget application, it is recommended that you use the variable\n`DAST_BROWSER_MAX_RESPONSE_SIZE_MB` to increase the limit for response\nsizes. The default is 10MB but can be increased for 50MB or more, if\nnecessary.\n\n\n## Tools for troubleshooting\n\n\nSeveral tools will help with DAST scan troubleshooting:\n\n\n- [authentication\nreport](https://docs.gitlab.com/ee/user/application_security/dast/authentication.html#configure-the-authentication-report)\n- This report can be produced during the scan and saved as a CI/CD job\nartifact to assist with understanding the cause of an authentication\nfailure. The report contains steps performed during the login process, HTTP\nrequests and responses, the Document Object Model (DOM), and screenshots. To\nconfigure the report, set `DAST_AUTH_REPORT` to `true` and configure an\nartifacts attribute for the DAST job, e.g.:\n\n\n```\n\ndast:\n   variables:\n      DAST_WEBSITE: \"https://example.com\"\n      DAST_AUTH_REPORT: \"true\"\n    artifacts:\n      paths: [gl-dast-debug-auth-report.html]\n      when: always\n```\n\n\n- [analyzer\nlogs](https://docs.gitlab.com/ee/user/application_security/dast/browser_based_troubleshooting.html#browser-based-analyzer-logging)\n- Setting `DAST_BROWSER_LOG` to `auth:debug` or `auth:trace` will provide\nadditional logging that may help identify an issue with the scan. \n\n\nThe browser-based DAST scan configuration depends on the specific attributes\nof the web application you're testing, including how authentication is\nimplemented to access the web site, what buttons are used, and how fast your\nbrowser loads once the user has authenticated. Using the appropriate\nvariables to guide the analyzer through the authentication process will\nensure that you are able to run a successful scan. And robust error logging\nand the authentication report will provide additional pointers to where the\nconfiguration might be incorrect and need to be adjusted.\n\n\nTry DAST scanning with [a free trial of GitLab\nUltimate](https://about.gitlab.com/free-trial/).\n\n\n## Read more\n\n- [How to configure DAST full scans for complex web\napplications](https://about.gitlab.com/blog/how-to-configure-dast-full-scans-for-complex-web-applications/)\n\n- [How we're using DAST 2 for easier scan\nconfiguration](https://about.gitlab.com/blog/how-were-using-dast2-for-easier-scan-configuration/)\n",[830,1057,1057,9],{"slug":4211,"featured":6,"template":700},"tips-to-configure-browser-based-dast-scans","content:en-us:blog:tips-to-configure-browser-based-dast-scans.yml","Tips To Configure Browser Based Dast Scans","en-us/blog/tips-to-configure-browser-based-dast-scans.yml","en-us/blog/tips-to-configure-browser-based-dast-scans",{"_path":4217,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4218,"content":4224,"config":4229,"_id":4231,"_type":13,"title":4232,"_source":15,"_file":4233,"_stem":4234,"_extension":18},"/en-us/blog/top-10-gitlab-hacks",{"title":4219,"description":4220,"ogTitle":4219,"ogDescription":4220,"noIndex":6,"ogImage":4221,"ogUrl":4222,"ogSiteName":685,"ogType":686,"canonicalUrls":4222,"schema":4223},"Top ten GitLab hacks for all stages of the DevOps Platform","Get the most out of the GitLab DevOps Platform with our ten best tips for enhanced productivity.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667482/Blog/Hero%20Images/cover-image-unsplash.jpg","https://about.gitlab.com/blog/top-10-gitlab-hacks","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Top ten GitLab hacks for all stages of the DevOps Platform\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2021-10-19\",\n      }",{"title":4219,"description":4220,"authors":4225,"heroImage":4221,"date":4226,"body":4227,"category":1119,"tags":4228},[1646],"2021-10-19","It's been ten years since the first commit to GitLab, so we are sharing our\nten favorite GitLab hacks to help you get the most out of our DevOps\nPlatform. These are tips for all stages of the development lifecycle, so\nroll up your sleeves and let's get started.\n\n\n## Manage faster with quick actions\n\n\nYou might have adopted keyboard shortcuts for faster navigation and\nworkflows already - if not, check out the GitLab documentation for [platform\nspecific shortcuts](https://docs.gitlab.com/ee/user/shortcuts.html). The\nknowledge of pressing `r` to land in the reply to comment in text form can\nbe combined with other quick actions, including:\n\n\n```\n\n/assign_reviewer @ \u003Csearch username>\n\n\n/label ~ \u003Csearch label>\n\n/label ~enhancement ~workflow::indev\n\n\n/due Oct 8\n\n\n/rebase\n\n\n/approve\n\n\n/merge \n\n```\n\n\nQuick actions are also helpful if you have to manage many issues, merge\nrequests and epics at the same time. There are specific actions which allow\nyou to duplicate existing issues, as one example. \n\n\nTake a deeper dive into [Quick\nActions](/blog/improve-your-gitlab-productivity-with-these-10-tips/). \n\n\n## Plan instructions with templates\n\n\nDon’t fall into the trap of back-and-forth with empty issue descriptions\nthat leave out details your development teams need to reproduce the error in\nthe best way possible. \n\n\nGitLab provides the possibility to use so-called [description\ntemplates](https://docs.gitlab.com/ee/user/project/description_templates.html)\nin issues and merge requests. Next to providing a structured template with\nheadings, you can also add [task\nlists](https://docs.gitlab.com/ee/user/markdown.html#task-lists) which can\nlater be ticked off by the assignee. Basically everything is possible and is\nsupported in GitLab-flavored markdown and HTML.\n\n\nIn addition to that, you can combine the static description templates with\nquick actions. This allows you to automatically set labels, assignees,\ndefine due dates, and more to level up your productivity with GitLab. \n\n\n```\n\n\u003C!-- \n\nThis is a comment, it will not be rendered by the Markdown engine. You can\nuse it to provide instructions how to fill in the template.\n\n--> \n\n\n### Summary \n\n\n\u003C!-- Summarize the bug encountered concisely. -->\n\n\n### Steps to reproduce\n\n\n\u003C!-- Describe how one can reproduce the issue - this is very important. -->\n\n\n### Output of checks\n\n\n\u003C!-- If you are reporting a bug on GitLab.com, write: This bug happens on\nGitLab.com -->\n\n\n#### Results of GitLab environment info\n\n\n\u003C!--  Input any relevant GitLab environment information if needed. -->\n\n\n\u003Cdetails>\n\n\u003Csummary>Expand for output related to app info\u003C/summary>\n\n\n\u003Cpre>\n\n\n(Paste the version details of your app here)\n\n\n\u003C/pre>\n\n\u003C/details>\n\n\n### Possible fixes\n\n\n\u003C!-- If you can, link to the line of code and suggest actions. →\n\n\n## Maintainer tasks\n\n\n- [ ] Problem reproduced\n\n- [ ] Weight added\n\n- [ ] Fix in test\n\n- [ ] Docs update needed\n\n\n/label ~\"type::bug\"\n\n```\n\n\nWhen you manage different types of templates, you can pass along the name of\nthe template in the `issuable_template` parameter, for example\n`https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal%20%23%20lean`. \n\n\nAt GitLab, we use description and merge request templates in many ways:\n[GitLab the\nproject](https://gitlab.com/gitlab-org/gitlab/-/tree/master/.gitlab/issue_templates),\n[GitLab Corporate Marketing\nteam](https://gitlab.com/gitlab-com/marketing/corporate_marketing/corporate-marketing/-/tree/master/.gitlab/issue_templates),\n[GitLab team member\nonboarding](https://gitlab.com/gitlab-com/people-group/people-operations/employment-templates/-/tree/master/.gitlab/issue_templates)\nand [GitLab product\nteam](https://gitlab.com/gitlab-com/Product/-/tree/main/.gitlab/issue_templates)\nare just a few examples.\n\n\n## Create with confidence \n\n\nWhen reading GitLab issues and merge requests, you may see the abbreviation\n`MWPS` which means `Merge When Pipeline Succeeds`. This is an efficient way\nto merge the MRs when the pipeline passes all jobs and stages - you can even\ncombine this workflow with [automatically closing\nissues](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)\nwith keywords from the MR.\n\n\n`Merge When Pipeline Succeeds` also works on the CLI with the `git` command\nand [push\noptions](https://docs.gitlab.com/ee/user/project/push_options.html). That\nway you can create a merge request from a local Git branch, and set it to\nmerge when the pipeline succeeds.\n\n\n```shell\n\n# mwps BRANCHNAME\n\nalias mwps='git push -u origin -o merge_request.create -o\nmerge_request.target=main -o merge_request.merge_when_pipeline_succeeds'\n\n```\n\n\nCheckout [this ZSH alias\nexample](https://gitlab.com/sytses/dotfiles/-/blob/745ef9725a859dd759059f6ce283e2a8132c9b00/git/aliases.zsh#L24)\nin our CEO [Sid Sijbrandij](/company/team/#sytses)’s dotfiles repository.\nThere are more push options available, and even more Git CLI tips in [our\ntools & tips\nhandbook](https://handbook.gitlab.com/handbook/tools-and-tips/#terminal).\nOne last tip: Delete all local branches where the remote branch was deleted,\nfor example after merging a MR.\n\n\n```shell\n\n# Delete all remote tracking Git branches where the upstream branch has been\ndeleted\n\nalias git_prune=\"git fetch --prune && git branch -vv | grep 'origin/.*:\ngone]' | awk '{print \\$1}' | xargs git branch -d\"\n\n```\n\n\nYou are not bound to your local CLI environment; take it to the cloud with\n[Gitpod](/blog/teams-gitpod-integration-gitlab-speed-up-development/)\nand either work in VS Code or the pod terminal. \n\n\n## Verify your CI/CD pipeline\n\n\nRemember the old workflow of committing a change to `.gitlab-ci.yml` just to\nsee if it was valid, or if the job template really inherits all the\nattributes? This has gotten a whole lot easier with our new [pipeline\neditor](https://docs.gitlab.com/ee/ci/pipeline_editor/). Navigate into the\n`CI/CD` menu and start building CI/CD pipelines right away.\n\n\nBut the editor is more than just another YAML editor. You’ll get live\nlinting, allowing you to know if there is a missing dash for array lists or\na wrong keyword in use before you commit. You can also preview jobs and\nstages or asynchronous dependencies with `needs` to make your pipelines more\nefficient.\n\n\nThe pipeline editor also uses uses the `/ci/lint` API endpoint, and fetches\nthe merged YAML configuration I described earlier in [this blog post about\njq and CI/CD\nlinting](/blog/devops-workflows-json-format-jq-ci-cd-lint/). That\nway you can quickly verify that job templates with\n[extends](https://docs.gitlab.com/ee/ci/yaml/#extends) and [!reference\ntags](https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags)\nwork in the way you designed them. It also allows you to unfold included\nfiles, and possible job overrides (for example changing the stage of an\n[included SAST security\ntemplate](https://docs.gitlab.com/ee/user/application_security/sast/#overriding-sast-jobs)).\n\n\nLet’s try a quick example – create a new project and new file called\n`server.c` with the following content: \n\n\n```\n\n#include \u003Cstdio.h>\n\n#include \u003Cstring.h>\n\n#include \u003Csys/mman.h>\n\n#include \u003Csys/stat.h>\n\n#include \u003Cunistd.h>\n\n\nint main(void) {\n    size_t pagesize = getpagesize();\n    char * region = mmap(\n        (void*) (pagesize * (1 \u003C\u003C 20)),\n        pagesize,\n        PROT_READ|PROT_WRITE|PROT_EXEC,\n        MAP_ANON|MAP_PRIVATE, 0, 0);\n\n    strcpy(region, \"Hello GitLab SAST!\");\n    printf(\"Contents of region: %s\\n\", region);\n\n    FILE *fp;\n    fp = fopen(\"devops.platform\", \"r\");\n    fprintf(fp, \"10 years of GitLab 🦊 🥳\");\n    fclose(fp);\n    chmod(\"devops.platform\", S_IRWXU|S_IRWXG|S_IRWXO);\n\n    return 0;\n}\n\n```\n\n\nOpen the CI/CD pipeline editor and add the following configuration, with an\nextra `secure` stage assigned to the `semgrep-sast` job for SAST and the C\ncode. \n\n\n```yaml\n\nstages:\n    - build\n    - secure\n    - test\n    - deploy\n\ninclude:\n    - template: Security/SAST.gitlab-ci.yml\n\nsemgrep-sast:\n    stage: secure\n```\n\n\nInspect the `Merged YAML tab` to see the fully compiled CI/CD configuration.\nYou can commit the changes and check the found vulnerabilities too as an\nasync practice :). The examples are available in [this\nproject](https://gitlab.com/gitlab-de/playground/sast-10y-example).\n\n\n![CI/CD Pipeline editor - Merged\nYAML](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_pipeline_editor_view_merged_yaml.png)\n\nVerify the stage attribute for the job by opening the `view merged YAML` tab\nin the CI/CD pipeline editor.\n\n{: .note.text-center}\n\n\n## Package your applications\n\n\nThe [package registry](https://docs.gitlab.com/ee/user/packages/)\npossibilities are huge and there are more languages and package managers to\ncome. Describing why Terraform, Helm, and containers (for infrastructure)\nand Maven, npm, NuGet, PyPI, Composer, Conan, Debian, Go and Ruby Gems (for\napplications) are so awesome would take too long, but it's clear there are\nplenty of choices. \n\n\nOne of my favourite workflows is to use existing CI/CD templates to publish\ncontainer images in the GitLab container registry. This makes continuous\ndelivery much more efficient, such as when deploying the application into\nyour Kubernetes cluster or AWS instances. \n\n\n```yaml\n\ninclude:\n  - template: 'Docker.gitlab-ci.yml'\n```\n\n\nIn addition to including the CI/CD template, you can also override the job\nattributes and define a specific stage and manual non-blocking rules.\n\n\n```yaml\n\nstages:\n  - build\n  - docker-build\n  - test\n\ninclude:\n  - template: 'Docker.gitlab-ci.yml'\n\n# Change Docker build to manual non-blocking\n\ndocker-build:\n  stage: docker-build\n  rules:\n    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'\n      when: manual \n      allow_failure: true\n```\n\n\nFor celebrating #10YearsOfGitLab, we have created a [C++\nexample](https://gitlab.com/gitlab-de/cicd-tanuki-cpp) with an Easter egg on\ntime calculations. This project also uses a Docker builder image to showcase\na more efficient pipeline. Our recommendation is to learn using the\ntemplates in a test repository, and then create a dedicated group/project\nfor managing all required container images. You can think of builder images\nwhich include the compiler tool chain, or specific scripts to run end-to-end\ntests, etc. \n\n\n## Secure your secrets\n\n\nIt is easy to leak a secret by making choices that uncomplicate a unit test\nby running it directly with the production database. The secret persists in\ngit history, and someone with bad intentions gains access to private data,\nor finds ways to exploit your supply chain even further. \n\n\nTo help prevent that, include the CI/CD template for secret detection. \n\n\n```yaml\n\nstages:\n    - test\n\ninclude:\n  - template: Security/Secret-Detection.gitlab-ci.yml  \n```\n\n\nA known way to leak secrets is committing the `.env` file which stores\nsettings and secrets in the repository. Try the following snippet by adding\na new file `.env` and create a merge request.\n\n\n```\n\nexport AWS_KEY=\"AKIA1318109798ABCDEF\"\n\n```\n\n\nInspect the reports JSON to see the raw reports structure. GitLab Ultimate\nprovides an MR integration, a security dashboard overview, and more features\nto take immediate action. The example can be found in [this\nproject](https://gitlab.com/gitlab-de/playground/secret-scanning-10y-example).\n\n\n![Secrets Scanning in\nMR](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_secrets_scanning.png)\n\nMR detail view with detected AWS secret from security scanning\n\n{: .note.text-center}\n\n\n## Release and continuously deliver (CD)\n\n\nGitLab’s release stage provides many\n[features](https://about.gitlab.com/handbook/product/categories/features/#release),\nincluding [canary\ndeployments](https://docs.gitlab.com/ee/user/project/canary_deployments.html)\nand [GitLab pages](https://docs.gitlab.com/ee/user/project/pages/). There\nare also infrastructure deployments with Terraform and cloud native\n(protected) [environments](https://docs.gitlab.com/ee/ci/environments/). \n\n\nWhile working on a CI/CD pipeline efficiency workshop, I got enthusiastic\nabout [parent-child\npipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html#parent-child-pipelines)\nallowing non-blocking child pipelines into production, with micro services\nin Kubernetes as one example. \n\n\nLet’s try it! Create a new project, and add 2 child pipeline configuration\nfiles: `child-deploy-staging.yml` and `child-deploy-prod.yml`. The naming is\nimportant as the files will be referenced in the main `.gitlab-ci.yml`\nconfiguration file later. The jobs in the child pipelines will sleep for 60\nseconds to simulate a deployment. \n\n\nchild-deploy-staging.yml:\n\n\n```yaml\n\ndeploy-staging:\n    stage: deploy\n    script:\n        - echo \"Deploying microservices to staging\" && sleep 60\n```\n\n\nchild-deploy-prod.yml\n\n\n```yaml\n\ndeploy-prod:\n    stage: deploy\n    script:\n        - echo \"Deploying microservices to prod\" && sleep 60\n\nmonitor-prod:\n    stage: deploy\n    script:\n        - echo \"Monitoring production SLOs\" && sleep 60\n```\n\n\nNow edit the `.gitlab-ci.yml` configuration file and create a\nbuild-test-deploy stage workflow.\n\n\n```yaml\n\nstages:\n  - build\n  - test\n  - deploy\n\nbuild:\n  stage: build\n  script: echo \"Build\"\n\ntest:\n  stage: test \n  script: echo \"Test\"\n\ndeploy-staging-trigger:\n  stage: deploy\n  trigger:\n    include: child-deploy-staging.yml\n  #rules:\n  #  - if: $CI_MERGE_REQUEST_ID\n\ndeploy-prod-trigger:\n  stage: deploy\n  trigger:\n    include: child-deploy-prod.yml\n    #strategy: depend\n  #rules:\n  #  - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH   \n```\n\n\nCommit the changes and inspect the CI/CD pipelines. \n\n\n![Parent-child\nPipelines](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_parent_child_pipelines.png)\n\nView parent-child pipelines in GitLab\n\n{: .note.text-center}\n\n\n`strategy: depends` allows you to make the child pipelines blocking again,\nand the parent child pipeline waits again. Try uncommenting this for the\nprod job, and verify that by inspecting the pipeline view.\n[Rules](https://docs.gitlab.com/ee/ci/yaml/#rules) allow refining the scope\nwhen jobs are being run, such as when staging child pipelines that should\nonly be run in merge requests and the prod child pipeline only gets\ntriggered when on the default main branch. The full example can be found in\n[this\nproject](https://gitlab.com/gitlab-de/playground/parent-child-pipeline-10y-example).\n\n\nTip: You can use\n[resource_groups](/blog/introducing-resource-groups/) to limit\nproduction deployments from running concurrent child pipelines. \n\n\n## Configure your infrastructure\n\n\nTerraform allows you to describe, plan and apply the provisioning of\ninfrastructure resources. The workflow requires a state file to be stored\nover steps, where the [managed state in\nGitLab](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html)\nas an HTTP backend is a great help, together with predefined container\nimages and CI/CD templates to make [Infrastructure as\ncode](https://docs.gitlab.com/ee/user/infrastructure/iac/) as smooth as\npossible.\n\n\nYou can customize the template, or copy the CI/CD configuration into\n.gitlab-ci.yml and modify the steps by yourself. Let’s try a quick example\nwith only an AWS account and an IAM user key pair. Configure them as CI/CD\nvariables in `Settings > CI/CD > Variables`: `AWS_ACCESS_KEY_ID` and\n`AWS_SECRET_ACCESS_KEY`.\n\n\nNext, create the `backend.tf` file and specify the http backend and AWS\nmodule dependency.\n\n\n```terraform\n\nterraform {\n  backend \"http\" {\n  }\n\n  required_providers {\n    aws = {\n      source = \"hashicorp/aws\"\n      version = \"~> 3.0\"\n    }\n  }\n}\n\n```\n\n\nCreate `provider.tf` to specify the AWS region.\n\n\n```terraform\n\nprovider \"aws\" {\n  region = \"us-east-1\"\n}\n\n```\n\n\nThe `main.tf` describes the S3 bucket resources.\n\n\n```terraform\n\nresource \"aws_s3_bucket_public_access_block\" \"publicaccess\" {\n  bucket = aws_s3_bucket.demobucket.id\n  block_public_acls = false\n  block_public_policy = false\n}\n\n\nresource \"aws_s3_bucket\" \"demobucket\" {\n  bucket = \"terraformdemobucket\"\n  acl = \"private\"\n}\n\n```\n\n\nTip: You can verify the configuration locally on your CLI by commenting out\nthe HTTP backend above.\n\n\nFor GitLab CI/CD, open the pipeline editor and use the following\nconfiguration: (Note that it is important to specify the `TF_ROOT` and\n`TF_ADDRESS` variables since you can [manage multiple Terraform state\nfiles](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html#configure-the-backend)). \n\n\n```yaml\n\nvariables:\n  TF_ROOT: ${CI_PROJECT_DIR}\n  TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME}\n\ninclude:\n    - template: Terraform.latest.gitlab-ci.yml\n\nstages:\n  - init\n  - validate\n  - build\n  - deploy\n  - cleanup\n\ndestroy:\n    stage: cleanup\n    extends: .terraform:destroy \n    when: manual\n    allow_failure: true\n```\n\n\nCommit the configuration and inspect the pipeline jobs. \n\n\n![Terraform pipeline AWS S3\nbucket](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_terraform_state_cicd_pipeline_aws_s3_bucket.png)\n\nAWS S3 bucket provisioned with Terraform in GitLab CI/CD \n\n{: .note.text-center}\n\n\nThe `destroy` job is not created in the template and therefore explicitly\nadded as a manual job. It is recommended to review the opinionated Terraform\nCI/CD template and copy the jobs into your own configuration to allow for\nfurther modifications or style adjustments.  The full example is located in\n[this\nproject](https://gitlab.com/gitlab-de/playground/terraform-aws-state-10y-example).\n\n\n![GitLab managed Terraform\nstates](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_terraform_state_cicd_overview.png)\n\nView the Terraform states in GitLab\n\n{: .note.text-center}\n\n\nHat tipping to our Package stage - you can manage and publish [Terraform\nmodules in the\nregistry](https://docs.gitlab.com/ee/user/packages/terraform_module_registry/)\ntoo, using all of the DevOps Platform advantages. And hot off the press, the\n[GitLab Kubernetes Operator is generally\navailable](/blog/open-shift-ga/). \n\n\n## Monitor GitLab and dive into Prometheus\n\n\nPrometheus is a monitoring solution which collects metrics from `/metrics`\nHTTP endpoints made available by applications, as well as so-called\nexporters to serve services and host information in the specified metrics\nformat. One example is CI/CD pipeline insights to analyse bottlenecks and\n[make your pipelines more\nefficient](https://docs.gitlab.com/ee/ci/pipelines/pipeline_efficiency.html).\nThe [GitLab CI Pipeline Exporter\nproject](https://github.com/mvisonneau/gitlab-ci-pipelines-exporter/tree/main/examples/quickstart)\nhas a great quick start in under 5 minutes, bringing up demo setup with\nDocker-compose, Prometheus and Grafana. From there, it is not far into your\nproduction monitoring environment, and monitoring more of GitLab. \n\n\n![GitLab CI\nExporter](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_ci_pipeline_exporter_prometheus.png)\n\nExample dashboard for the GitLab CI Pipeline Exporter\n\n{: .note.text-center}\n\n\nThe Prometheus Exporter uses the [Go client\nlibraries](https://prometheus.io/docs/instrumenting/writing_exporters/).\nThey can be used to write your own exporter, or instrument your application\ncode to expose `/metrics`. When deployed, you can use Prometheus again to\nmonitor the performance of your applications in Kubernetes, as one example.\nFind more monitoring ideas in my talk “[From Monitoring to Observability:\nLeft Shift your\nSLOs](https://docs.google.com/presentation/d/1LPb-HPMgbc8_l98VjMEo5d0uYlnNnAtJSURngZPWDdE/edit)”. \n\n\n## Protect\n\n\nYou can enable security features in GitLab by including the CI/CD templates\none by one. A more easy way is to enable [Auto\nDevOps](https://docs.gitlab.com/ee/topics/autodevops/) and use the default\nbest practices for [security\nscans](https://docs.gitlab.com/ee/user/application_security/index.html#security-scanning-with-auto-devops).\nThis includes [container\nscanning](https://docs.gitlab.com/ee/topics/autodevops/stages.html#auto-container-scanning)\nensuring that application deployments are not vulnerable on the container OS\nlevel. \n\n\nLet’s try a quick example with a potentially vulnerable image, and the\nDocker template tip from the Package stage above. Create a new `Dockerfile`\nin a new project:\n\n\n```yaml\n\nFROM debian:10.0 \n\n```\n\n\nOpen the pipeline editor and add the following CI/CD configuration:\n\n\n```yaml\n\n# 1. Automatically build the Docker image\n\n# 2. Run container scanning.\nhttps://docs.gitlab.com/ee/user/application_security/container_scanning/index.html\n\n# 3. Inspect `Security & Compliance > Security Dashboard`\n\n\n# For demo purposes, scan the latest tagged image from 'main'\n\nvariables:\n    DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest    \n\ninclude:\n    - template: Docker.gitlab-ci.yml\n    - template: Security/Container-Scanning.gitlab-ci.yml\n```\n\n\nThe full example is located in [this\nproject](https://gitlab.com/gitlab-de/playground/container-scanning-10y-example).\n\n\nTip: Learn more about [scanning container images in a deployed Kubernetes\ncluster](https://docs.gitlab.com/ee/user/application_security/container_scanning/)\nto stay even more safe. \n\n\n![Container Scanning Vulnerability\nReport](https://about.gitlab.com/images/blogimages/top-10-gitlab-hacks/gitlab_10y_container_scanning_vulnerability_report.png)\n\nView the container scanning vulnerability report\n\n{: .note.text-center}\n\n\n## What’s next?\n\n\nWe have tried to find a great “hack” for each stage of the DevOps lifecycle.\nThere are more hacks and hidden gems inside GitLab - share yours and be\nready to explore more stages of the DevOps Platform.\n\n\nCover image by [Alin Andersen](https://unsplash.com/photos/diUGN5N5Rrs) on\n[Unsplash](https://unsplash.com)\n",[9,789,956],{"slug":4230,"featured":6,"template":700},"top-10-gitlab-hacks","content:en-us:blog:top-10-gitlab-hacks.yml","Top 10 Gitlab Hacks","en-us/blog/top-10-gitlab-hacks.yml","en-us/blog/top-10-gitlab-hacks",{"_path":4236,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4237,"content":4242,"config":4248,"_id":4250,"_type":13,"title":4251,"_source":15,"_file":4252,"_stem":4253,"_extension":18},"/en-us/blog/top-10-gitlab-technical-blogs-of-2023",{"title":4238,"description":4239,"ogTitle":4238,"ogDescription":4239,"noIndex":6,"ogImage":3141,"ogUrl":4240,"ogSiteName":685,"ogType":686,"canonicalUrls":4240,"schema":4241},"Top 10 GitLab technical blogs of 2023","2023 was a big year! Catch up on expert insights into DevSecOps, AI, CI/CD, and more.","https://about.gitlab.com/blog/top-10-gitlab-technical-blogs-of-2023","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Top 10 GitLab technical blogs of 2023\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sandra Gittlen\"}],\n        \"datePublished\": \"2024-01-09\",\n      }",{"title":4238,"description":4239,"authors":4243,"heroImage":3141,"date":4245,"body":4246,"category":1119,"tags":4247},[4244],"Sandra Gittlen","2024-01-09","2023 brought fresh insights from experts across GitLab and beyond —  all of them focused on the challenges and opportunities facing DevSecOps teams. From Lockheed Martin to CARFAX, organizations are trying to understand and unlock the power of technologies such as artificial intelligence (AI), CI/CD, security automation, and more. Our experts provided tips, best practices, and tutorials to use throughout the software development lifecycle.\n\nHere are the top 10 technical blogs from what was an incredible year in DevSecOps innovation.\n\n**1. [Jenkins to GitLab: The ultimate guide to modernizing your CI/CD environment](https://about.gitlab.com/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment/)**\nLooking for a smooth transition from Jenkins to GitLab? Follow this step-by-step tutorial to learn how GitLab's integrated CI/CD capabilities help deliver high-quality software faster.\n\n**2. [U.S. Navy Black Pearl: Lessons in championing DevSecOps](https://about.gitlab.com/blog/u-s-navy-black-pearl-lessons-in-championing-devsecops/)**\nSigma Defense's director of engineering details what it's like to manage the U.S. Navy's Black Pearl, which uses GitLab as its DevSecOps platform. The DevSecOps champion relays his experience implementing DevSecOps and the benefits of that decision.\n\n**3. [Quickstart guide for GitLab Remote Development workspaces](https://about.gitlab.com/blog/quick-start-guide-for-gitlab-workspaces/)**\nEnabling developers to work in their preferred environments empowers DevSecOps teams to build and deliver software more efficiently. With these quickstart instructions, developers can create a workspace, use the Web IDE Terminal to install dependencies or start their server, and view their running application.\n\n**4. [Introducing the GitLab CI/CD Catalog Beta](https://about.gitlab.com/blog/introducing-the-gitlab-ci-cd-catalog-beta/)**\nCI/CD catalogs are a game-changer, allowing developers to discover, integrate, and share pre-existing CI/CD components with ease. This tutorial shows how to get the most from this new DevSecOps platform feature.\n\n**5. [Combine GitLab Flow and GitLab Duo for a workflow powerhouse](https://about.gitlab.com/blog/gitlab-flow-duo/)**\nGitLab Flow and GitLab Duo can help organizations achieve significant improvements in end-to-end workflow efficiency that can lead to higher levels of productivity, deployment frequency, code quality and overall security, and production resiliency and availability. Find out how with this step-by-step guide.\n\n**6. [Efficient DevSecOps workflows: Hands-on python-gitlab API automation](https://about.gitlab.com/blog/efficient-devsecops-workflows-hands-on-python-gitlab-api-automation/)**\nThe python-gitlab library is a useful abstraction layer for the GitLab API. Dive into hands-on examples and best practices in this tutorial.\n\n**7. [Building GitLab with GitLab: Why there is no MLOps without DevSecOps](https://about.gitlab.com/blog/there-is-no-mlops-without-devsecops/)**\nAt GitLab, we believe in the power of MLOps, especially when combined with DevSecOps. So follow along as our data scientists adopt DevSecOps practices and enjoy the benefits of automation, repeatable workflows, standardization, and automatic provisioning of infrastructure.\n\n**8. [Explore the Dragon Realm: Build a C++ adventure game with a little help from AI](https://about.gitlab.com/blog/building-a-text-adventure-using-cplusplus-and-code-suggestions/)**\nReaders are invited to create a mystical world while learning how to integrate AI into their coding environment. This tutorial demonstrates how to use GitLab Duo Code Suggestions to create a text-based adventure game, including magical locations to visit and items to procure, using C++. \n\n**9. [How GitLab's Red Team automates C2 testing](https://about.gitlab.com/blog/how-gitlabs-red-team-automates-c2-testing/)**\nThe GitLab Red Team conducts security exercises that simulate real-world threats. They apply professional development practices to using the same open source C2 tools as threat actors. In this tutorial, the GitLab Red Team shares how they implement continuous testing for the Mythic framework, their design philosophy, and a public project that can be forked for use by other Red Teams.\n\n**10. [Building GitLab with GitLab: How GitLab.com inspired Dedicated](https://about.gitlab.com/blog/building-gitlab-with-gitlabcom-how-gitlab-inspired-dedicated/)**\nThe design of GitLab Dedicated, our single-tenancy SaaS version of the DevSecOps platform, came from the lessons learned while building GitLab.com. In this peek behind the curtains, learn the considerations that sparked different decisions regarding automation, databases, monitoring, availability, and more – and what the outcome was.\n\nSign up for the GitLab newsletter using the form to the right to receive the latest blogs right in your inbox.\n",[720,108,495,851,830,9],{"slug":4249,"featured":90,"template":700},"top-10-gitlab-technical-blogs-of-2023","content:en-us:blog:top-10-gitlab-technical-blogs-of-2023.yml","Top 10 Gitlab Technical Blogs Of 2023","en-us/blog/top-10-gitlab-technical-blogs-of-2023.yml","en-us/blog/top-10-gitlab-technical-blogs-of-2023",{"_path":4255,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4256,"content":4262,"config":4267,"_id":4269,"_type":13,"title":4270,"_source":15,"_file":4271,"_stem":4272,"_extension":18},"/en-us/blog/top-10-gitlab-workflow-hacks-you-need-to-know",{"title":4257,"description":4258,"ogTitle":4257,"ogDescription":4258,"noIndex":6,"ogImage":4259,"ogUrl":4260,"ogSiteName":685,"ogType":686,"canonicalUrls":4260,"schema":4261},"Top 10 GitLab workflow hacks you need to know","A GitLab product manager shares her favorite tricks to navigate quickly and efficiently around the GitLab DevSecOps Platform and to boost team collaboration.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099361/Blog/Hero%20Images/Blog/Hero%20Images/lightvisibility_lightvisibility.png_1750099361252.png","https://about.gitlab.com/blog/top-10-gitlab-workflow-hacks-you-need-to-know","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Top 10 GitLab workflow hacks you need to know\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Amanda Rueda\"}],\n        \"datePublished\": \"2024-04-09\",\n      }",{"title":4257,"description":4258,"authors":4263,"heroImage":4259,"date":4264,"body":4265,"category":1119,"tags":4266},[2717],"2024-04-09","In the world of software development, efficiency isn't just about moving fast – it's about smart navigation. As a GitLab product manager, I truly understand the value of efficiency when working within the DevSecOps platform. These are my top 10 favorite GitLab features and they might be the workflow hacks you never knew you needed.\n\nLet's dive into these hidden gems to unlock a new level of productivity and collaboration within your team.\n\n## 1. Resolve comments\n\nNot just for merge requests! Resolving comments on issues can significantly reduce noise and streamline task management. It's particularly handy for managing feedback efficiently.\n\n> **Why do I love it?** Not only does resolving comments reduce the noise on an issue, but it’s also a great way to manage tasks.\n>\n> **Use case.** Resolving comments is a great tool for issues where you are collecting feedback – respond to the feedback and provide a link, resolve the comment, and move on to the next one.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/discussions/#resolve-a-thread)__\n\n![example of resolve comments - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750099376147.gif)\n\n\u003Cp>\u003C/p>\n\n## 2. Internal comments\n\nSpeak directly to your team without an external audience. Keep discussions private within an issue or merge request with comments visible only to your team members. It's the perfect balance between transparency and privacy.\n\n> **Why do I love it?** It balances privacy with transparency, while keeping the broader discussion open for the community.\n>\n> **Use case.** When coordinating a product launch, your marketing team can use internal comments to discuss and refine messaging and strategy. This keeps your discussions centralized and easily accessible to the team while in draft mode.\n>\n> **[How-to documentation](https://docs.gitlab.com/ee/user/discussions/#add-an-internal-note)**\n\n![internal comments example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099376148.png)\n\n\u003Cp>\u003C/p>\n\n## 3. And/or in filters\n\nWhen searching records on a listing page, using and/or filters can help you slice through the noise and find exactly what you're looking for quickly and efficiently.\n\n> **Why do I love it?** Perfect for finding exactly what you need, powering efficient and streamlined workflows.\n>\n>**Use case.** Search for feature issues related to a specific initiative that are assigned to specific groups.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#filter-with-the-or-operator)__\n\n![and/or filter example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/and_or__1__aHR0cHM6_1750099376152.gif)\n\n\u003Cp>\u003C/p>\n\n## 4. Auto expand URLs\n\nAppending '+' or '+s' to the end of a GitLab URL transforms it into an informative snippet, allowing you to share progress without forcing your teammates to leave the page.\n\n> **Why do I love it?** It's like having x-ray vision for URLs – see the important stuff without even clicking!\n>\n> **Use case.** Sharing progress in comments? Just add '+s' to the link, and boom – everyone's instantly on the same page.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/markdown.html#show-the-issue-merge-request-or-epic-title-in-the-reference)__\n\n![auto expand URLs example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750099376154.gif)\n\n\u003Cp>\u003C/p>\n\n## 5. Quick actions\n\nWith simple text commands, quick actions let you perform tasks like assigning users, adding labels, and more, directly from the description or comment box, saving you clicks and time.\n\n> **Why do I love it?** Saves clicks and time.\n>\n> **Use case.** When creating a new issue I use quick actions to automatically add labels, a milestone, and connect to the epic upon saving the record.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/project/quick_actions.html)__\n\n![quick actions example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750099376156.gif)\n\n\u003Cp>\u003C/p>\n\n## 6. Bulk edit\n\nApply labels, change assignees, or update milestones for multiple issues at once. This feature turns potentially tedious updates into a breeze, allowing for quick adjustments across numerous issues.\n\n> **Why do I love it?** Because it turns tedious updates into quick updates!\n>\n> **Use case.** Need to tag the whole sprint's issues as Review needed? Just filter, select all, and add that label in bulk – easy peasy.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#bulk-edit-issues-from-a-project)__\n\n![bulk edit example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750099376157.gif)\n\n\u003Cp>\u003C/p>\n\n## 7. Epic swimlanes\n\nGroup issues under epics on your board to visually track and discuss progress. It's a powerful way to contextualize work during reviews or standups.\n\n> **Why do I love it?** Easily understand the context of work as you’re walking the board.\n>\n> **Use case.** Group by epic during standup reviews to easily piece together work with its parent initiative.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/project/issue_board.html#group-issues-in-swimlanes)__\n\n![epic swimlanes example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750099376158.gif)\n\n\u003Cp>\u003C/p>\n\n## 8. Wiki diagrams\n\nIllustrate ideas and workflows directly in your wiki pages with easy-to-create diagrams. This feature supports visual learning and simplifies complex concepts.\n\n> **Why do I love it?** It’s incredibly user-friendly and flexible.\n>\n> **Use case.** When outlining a new feature workflow, draw it directly in the wiki page, making it crystal clear for everyone on the team.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/administration/integration/diagrams_net.html)__\n\n![wiki diagrams example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750099376159.gif)\n\n\u003Cp>\u003C/p>\n\n## 9. Table creation\n\nForget about wrestling with markdown for table creation. The rich text editor lets you effortlessly insert and format tables, making documentation cleaner and more structured.\n\n> **Why do I love it?** It turns the table creation ordeal into a breeze, making updates clean and structured with just a few clicks.\n>\n> **Use case.** Compiling a sprint retro? Quickly insert a table to organize feedback, action items, and owners, making the review process smoother for everyone.\n>\n> __[How-to documentation](https://docs.gitlab.com/ee/user/rich_text_editor.html#tables)__\n\n![table creation example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750099376160.gif)\n\n\u003Cp>\u003C/p>\n\n## 10. Video and GIF embeds\n\nEnhance your issues and epic descriptions or comments with embedded GIFs and YouTube videos, adding a dynamic layer to your communication.\n\n> **Why do I love it?** Sometimes a GIF or video speaks better than words.\n>\n> **Use case.** Trying to explain a UI bug? Embed a YouTube video for a quick walkthrough of the proposed feature enhancement.\n\n![video and gif embed example](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099376/Blog/Content%20Images/Blog/Content%20Images/gif__1__aHR0cHM6_1750099376161.gif)\n\n\u003Cp>\u003C/p>\n\n## Explore these features\n\nThese features represent just the tip of the iceberg in GitLab's comprehensive toolkit designed to boost efficiency and foster better collaboration. While they may be underutilized, their impact on your workflow could be substantial. I encourage you to explore these features further and integrate them into your daily routines.\n\n> Are you excited to power your DevSecOps workflow using GitLab? [Try GitLab Ultimate for free for 30 days](https://gitlab.com/-/trial_registrations/new).\n",[9,495,696,956],{"slug":4268,"featured":6,"template":700},"top-10-gitlab-workflow-hacks-you-need-to-know","content:en-us:blog:top-10-gitlab-workflow-hacks-you-need-to-know.yml","Top 10 Gitlab Workflow Hacks You Need To Know","en-us/blog/top-10-gitlab-workflow-hacks-you-need-to-know.yml","en-us/blog/top-10-gitlab-workflow-hacks-you-need-to-know",{"_path":4274,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4275,"content":4281,"config":4286,"_id":4288,"_type":13,"title":4289,"_source":15,"_file":4290,"_stem":4291,"_extension":18},"/en-us/blog/top-10-technical-articles-of-2022",{"title":4276,"description":4277,"ogTitle":4276,"ogDescription":4277,"noIndex":6,"ogImage":4278,"ogUrl":4279,"ogSiteName":685,"ogType":686,"canonicalUrls":4279,"schema":4280},"Top 10 technical articles of 2022","Let’s review our fantastic year of how-to guides. From fixing failed pipelines to making the best use of GitOps, we have you covered with our in-depth tutorials.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749663736/Blog/Hero%20Images/a-deep-dive-into-the-security-analyst-persona.jpg","https://about.gitlab.com/blog/top-10-technical-articles-of-2022","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Top 10 technical articles of 2022\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Valerie Silverthorne\"}],\n        \"datePublished\": \"2022-12-08\",\n      }",{"title":4276,"description":4277,"authors":4282,"heroImage":4278,"date":4283,"body":4284,"category":785,"tags":4285},[3777],"2022-12-08","\nWith 2022 coming to a close, we wanted to ensure everyone gets one more chance to explore our top 10 technical blog posts of the year. Roll up your sleeves and enjoy our most-viewed how-to articles and don’t forget to bookmark them for next year!\n\n## 1. Failed pipeline?\n\nWe have *all* been there, and not much is more frustrating than that red X. Staff Developer Evangelist [Brendan O’Leary](/company/team/#brendan) offers his best advice on troubleshooting the “why?” of a GitLab failed pipeline – it starts with keeping the right perspective. So many factors are involved in code development that it’s critical to ask all of the questions: Is it the code? Is it the test? Is it a vulnerability, etc.?\n\n[How to troubleshoot a GitLab pipeline failure](https://docs.gitlab.com/ci/debugging/)\n\n## 2. Why Git Rebase is your BFF\n\nWith code review increasingly important to successful DevOps, Senior Backend Engineer (Gitaly) [Christian Couder](/company/team/#chriscool) thinks devs might be forgetting a secret weapon in their IDE: Git Rebase. Learn how to rework commits with Git Rebase, including expert tips to try different instructions like ‘reword’, ‘edit’, and ‘squash’.\n\n[Take advantage of Git Rebase](/blog/take-advantage-of-git-rebase/)\n\n## 3. Alert fatigue is real\n\nFollow along with Senior Site Reliability Engineer [Steve Azzopardi](/company/team/#steveazz) as he lays out a GitLab investigation into annoying, time-consuming (and customer-facing) 502 errors in the GitLab Pages logs. To uncover the problem, Azzopardi and team had to unearth some red herrings along the way, but ultimately discovered the importance of PID 1 in a container.\n\n[How we reduced 502 errors by caring about PID 1 in containers](/blog/how-we-removed-all-502-errors-by-caring-about-pid-1-in-kubernetes/)\n\n## 4. More pipelines = less complexity\n\nCI/CD is at the heart of most modern DevOps practices, but that doesn’t mean it’s a “set it and forget it.” Staff Backend Engineer Fabio Pittino acknowledges the complexity challenges of CI/CD and suggests the solution is choosing the right pipelines for the job. Understand the differences between parent-child and multi-project pipelines to streamline your CI/CD efforts.\n\n[Breaking down CI/CD complexity with parent-child and multi-project pipelines](/blog/parent-child-vs-multi-project-pipelines/)\n\n## 5. Hacking and bug bounties\n\nHow did a Swedish web developer go from zero to number seven on our HackerOne Top 10 list in just over a year? Johan Carlsson offers a detailed look at how and why he started looking for bugs in GitLab in his spare time, and how others can jump into hacking, too.\n\n[Want to start hacking? Here’s how to quickly dive in](/blog/cracking-our-bug-bounty-top-10/)\n\n## 6. Gitlab… on an iPad\n\nYes, you can code on an M1-chip-based iPad, and Staff Developer Evangelist Brendan O’Leary walks through all the necessary steps to get GitLab running using GitPod.\n\n[How to code, build, and deploy from an iPad using GitLab and GitPod](/blog/how-to-code-build-and-deploy-from-an-ipad-using-gitlab-and-gitpod/)\n\n## 7. Speed up database changes\n\nMany DevOps teams have mastered speedy application code changes but have struggled to make database updates equally streamlined. In this step-by-step guide, you’ll learn how to apply DevOps principles to database change management.\n\n[How to bring DevOps to the database with GitLab and Liquibase](/blog/how-to-bring-devops-to-the-database-with-gitlab-and-liquibase/)\n\n## 8. A primer on IaC security\n\nInfrastructure as Code (IaC) is an increasingly popular solution for DevOps teams, and with good reason: It’s an efficient and low-resource solution. But, as Senior Developer Evangelist [Michael Friedrich](/company/team/#dnsmichi) explains, it’s also ripe with potential security vulnerabilities. Friedrich takes an exhaustive look at the threats, tools, integrations, and strategies that make IaC a safer choice.\n\n[Fantastic Infrastructure as Code security attacks and how to find them](/blog/fantastic-infrastructure-as-code-security-attacks-and-how-to-find-them/)\n\n## 9. Everything you need to know about GitOps\n\nWant to know how to make GitLab work with GitOps? Senior Product Manager (Configure) [Viktor Nagy](/company/team/#nagyv-gitlab) created an eight-part tutorial covering everything GitLab and GitOps, culminating in how to make a GitLab agent for Kubernetes self-managing.\n\n[The ultimate guide to GitOps with GitLab](/blog/the-ultimate-guide-to-gitops-with-gitlab/)\n\n## 10. The skinny on static site generators\n\nDevs will get the most out of GitLab Pages by choosing the right static site generator (SSG). Developer Evangelist [Fatima Sarah Khalid](/company/team/#sugaroverflow) reviews six options and has also created a toolkit to help make the SSG evaluation process easier.\n\n[How to choose the right static site generator](/blog/comparing-static-site-generators/)\n\n",[789,9,742],{"slug":4287,"featured":6,"template":700},"top-10-technical-articles-of-2022","content:en-us:blog:top-10-technical-articles-of-2022.yml","Top 10 Technical Articles Of 2022","en-us/blog/top-10-technical-articles-of-2022.yml","en-us/blog/top-10-technical-articles-of-2022",{"_path":4293,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4294,"content":4300,"config":4305,"_id":4307,"_type":13,"title":4308,"_source":15,"_file":4309,"_stem":4310,"_extension":18},"/en-us/blog/top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo",{"title":4295,"description":4296,"ogTitle":4295,"ogDescription":4296,"noIndex":6,"ogImage":4297,"ogUrl":4298,"ogSiteName":685,"ogType":686,"canonicalUrls":4298,"schema":4299},"Top tips for efficient AI-powered Code Suggestions with GitLab Duo","Explore best practices  for using Code Suggestions and how to combine it with our other AI features to greatly improve the developer experience (includes real-world exercises).","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749669095/Blog/Hero%20Images/gitlabduo.png","https://about.gitlab.com/blog/top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Top tips for efficient AI-powered Code Suggestions with GitLab Duo\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2024-06-11\",\n      }",{"title":4295,"description":4296,"authors":4301,"heroImage":4297,"date":4302,"body":4303,"category":718,"tags":4304},[1646],"2024-06-11","[GitLab Duo](https://about.gitlab.com/gitlab-duo/), our suite of AI-powered features, provides a unique opportunity to make your DevSecOps workflows more efficient. To make the most of GitLab Duo requires hands-on practice and learning in public together. This tutorial centers on GitLab Duo Code Suggestions and provides tips and tricks, learned best practices, and some hidden gems (including how to pair Code Suggestions with our other AI features for even more efficiency). You'll also discover how AI greatly improves the developer experience.\n\nThe best practices, tips, and examples in this article have been created from scratch and are included in the [GitLab Duo documentation](https://docs.gitlab.com/ee/user/gitlab_duo/index.html) and [GitLab Duo prompts project](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts), maintained by the GitLab Developer Relations team. Bookmark this page, and navigate into the respective chapters at your convenience.\n\nWhat you'll learn:\n\n1. [Why use GitLab Duo Code Suggestions?](#why-use-gitlab-duo-code-suggestions%3F)\n1. [Start simple, refine prompts](#start-simple-refine-prompts)\n1. [Practice, practice, practice](#practice-practice%2C-practice)\n    - [Fix missing dependencies](#fix-missing-dependencies)\n    - [Boilerplate code: Optimized logging](#boilerplate-code-optimized-logging)\n    - [Utility helper functions, well tested](#utility-helper-functions-well-tested)\n    - [Generate regular expressions](#generate-regular-expressions)\n1. [Re-trigger Code Suggestions](#re-trigger-code-suggestions)\n    - [Common keyboard combinations to re-trigger Code Suggestions](#common-keyboard-combinations-to-re-trigger-code-suggestions)\n    - [Stuck in the middle of suggestions](#stuck-in-the-middle-of-suggestions)\n    - [Code Suggestions stopped](#code-suggestions-stopped)\n1. [Code Suggestions vs, code generation](#code-suggestions-vs.-code-generation)\n    - [Start with a comment on top for code generation](#start-with-a-comment-on-top-for-code-generation)\n    - [Intent detection for code suggestions and generation](#intent-detection-for-code-suggestions-and-generation)\n    - [Tell a story for efficient code generation](#tell-a-story-for-efficient-code-generation)\n    - [Generate regular expressions](#generate-regular-expressions)\n    - [Iterate faster with code generation](#iterate-faster-with-code-generation)\n    - [Practical code generation: Cloud-native observability](#practical-code-generation-cloud-native-observability)\n1. [Take advantage of all GitLab Duo features](#take-advantage-of-all-gitlab-duo-features)\n    - [Combine Chat with Code Suggestions](#combine-chat-with-code-suggestions)\n    - [Use Chat to generate build configuration](#use-chat-to-generate-build-configuration)\n    - [Use Chat to explain potential vulnerabilities](#use-chat-to-explain-potential-vulnerabilities)\n    - [Combine vulnerability resolution with Code Suggestions](#combine-vulnerability-resolution-with-code-suggestions)\n1. [More tips](#more-tips)\n    - [Verify code quality and security](#verify-code-quality-and-security)\n    - [Learn as a team, and understand AI's impact](#learn-as-a-team-and-understand-ai-impact)    \n    - [Development is a marathon, not a sprint](#development-is-a-marathon-not-a-sprint)\n    - [Contribute using GitLab Duo](#contribute-using-gitlab-duo)\n1. [Share your feedback](#share-your-feedback)\n\n## Why use GitLab Duo Code Suggestions?\n\nConsider these two scenarios:\n\n1. As a senior developer, you have the confidence in your ability with various programming languages to write new source code, review existing code, design resilient architectures, and implement new projects. However, getting familiar with the latest programming language features requires time, research, and a change of habits. So how can you quickly learn about new language feature additions that could make your code even more robust or use resources more sustainably?\n\n    - As a personal example, I learned the C++03 standard, later C++11 and never really touched base on C++14/17/20/23 standards. Additionally, new languages such as [Rust](https://about.gitlab.com/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/) came around and offered better developer experiences. What now?\n\n2. As a new engineer, it can be challenging to navigate new projects, get familiar with a new programming language, understand specific algorithms, and find the documentation for structures, interfaces, and other technical components. New engineers are also learning under pressure, which often leads to errors and roadblocks along the way. There is no time for digging into best practices.\n\n    - I, myself, never really learned frontend engineering, just some self-taught HTML, CSS, and JavaScript. Adapting into frontend frameworks such as VueJS after a decade feels overwhelming, and I have little time to learn.\n\nThese scenarios show how hard it can be to keep up with the latest programming languages, best practices, and other key information. [GitLab Duo Code Suggestions](https://about.gitlab.com/solutions/code-suggestions/), which predictively completes code blocks, defines function logic, generates tests, and proposes common code like regex patterns – all in your coding environment. Code Suggestions provides the AI assistance necessary to learn what you need to know while staying in your development flow.\n\n> Live demo! Discover the future of AI-driven software development with our GitLab 17 virtual launch event. [Register today](https://about.gitlab.com/seventeen/)!\n\n## Start simple, refine prompts\n\nMy own GitLab Duo adoption journey started with single-line code comments, leading to not-so-great results at first. \n\n```\n# Generate a webserver\n\n// Create a database backend\n\n/* Use multi-threaded data access here */\n```\n\nAfter experimenting with different contexts, and writing styles, I found that code generation out of refined comments worked better. \n\n```\n# Generate a webserver, using the Flask framework. Implement the / URL endpoint with example output.\n\n// Create a database backend. Abstract data handlers and SQL queries into function calls.\n\n/* Use multi-threaded data access here. Create a shared locked resource, and focus on supporting Linux pthreads. */\n```\n\nCode comments alone won't do the trick, though. Let's explore more best practices.\n\n## Practice, practice, practice \n\nFind use cases and challenges for your daily workflows, and exclusively use GitLab Duo. It can be tempting to open browser search tabs, but you can also solve the challenge in your IDE by using GitLab Duo. Here are some examples:\n\n1. Fix missing dependencies (which always cause build/execution failures).\n1. If you're missing logging context, let Code Suggestions auto-complete started function calls, including `print` statements.\n1. Generate common methods and attributes for object-oriented design patterns (e.g. getter/setter methods, `toString()` and object comparison operators, object inheritance, etc.).\n1. Identify the function that generates random crashes. Use Code Suggestions to implement a new function with a different algorithm.\n1. If you encounter application cannot be compiled or executed, cryptic error, ask GitLab Duo Chat about it.\n1. Learn about existing (legacy) code, and strategies to document and refactor code into modern libraries. Start a v2 of an application with a new framework or programming language, helping solve technical debt.\n1. Prevent operations and security issues in Git history by detecting them before they occur (e.g. performance, crashes, security vulnerabilities).\n\nThink of the most boring - or most hated - coding task, and add it to the list above. My least favorite tasks are attribute getter/setter methods in C++ classes (as can be seen in the video below), immediately followed by regular expressions for email address format.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Z9EJh0J9358?si=QGvQ6mXxPPz4WpM0\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\nIt can also help to use Code Suggestions in different programming languages, for example focusing on backend and frontend languages. If you are experienced in many languages, take a look into languages that you have not used in a while, or look into learning a new programming language such as [Python](https://about.gitlab.com/blog/learning-python-with-a-little-help-from-ai-code-suggestions/) or [Rust](https://about.gitlab.com/blog/learning-rust-with-a-little-help-from-ai-code-suggestions-getting-started/). \n\nWhen you adopt Code Suggestions into a fast auto-completion workflow, it can happen without any interruption. The suggested code is greyed out and optional, depending on the user interface – for example, VS Code. This means that it will not distract you from continuing to write source code. Try using Code Suggestions on your own by familiarizing yourself with how suggestions are shown, how you can fully or partially accept them, and soon they will become optional help to write better code. \n\n![Image with code suggestions greyed out](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_java_springboot_class_methods_tostring.png)\n\n### Fix missing dependencies\n\nAfter building or running source code, missing dependency errors might be logged and prevent further execution and testing. The following example in Go shows an error from `go build`, where the source code did not import any dependencies yet. A manual approach can be collecting all listed dependencies, running a unique sort on them, and adding them into the source code file, as shown below.\n\n![Go build failed - missing dependencies](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_build_failed_missing_deps.png)\n\nBut what if GitLab Duo knows about the file context and missing dependencies already? Navigate into the top section and add a comment, saying `// add missing imports` and wait for Code Suggestions.\n\n![GitLab Duo Code Suggestions - go build failed missing dependencies suggested fix](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_build_failed_missing_deps_suggested_fix.png)\n\nRunning `go build` again results in success, and the source code can be tested and run.\n\n![Go build failed - missing dependencies fixed](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_build_failed_missing_deps_fixed.png)\n\n### Boilerplate code: Optimized logging\n\n*Q: Logging – and more observability data with metrics and traces – can be hard and tedious to implement. What is the most efficient way to implement them that does not impact the application performance or cause bugs?*\n\n*A: Use Code Suggestions to generate logging function calls, and refactor the code into robust observability instrumentation library abstractions. This method can help to prepare the code for later integration with [OpenTelemetry](https://docs.gitlab.com/ee/development/stage_group_observability/gitlab_instrumentation_for_opentelemetry.html), for example.*\n\nExample for a logging class in Ruby:\n\n```ruby\n# Create Logging utility class\n# Define default log level as attribute\n# Add method for logging, inputs: level, app, message\n# Print the data with formatted date and time in syslog format\n\n# Potential AI-generated code below\nclass Logging\n  attr_accessor :log_level\n\n  def log(level, app, message)\n    time = Time.now.strftime(\"%b %d %H:%M:%S\")\n    puts \"#{time} #{app}[#{Process.pid}]: #{level} - #{message}\"\n  end\nend\n\n# Instantiate class and test the log method\n\n# Potential AI-generated code below\nlogger = Logging.new\nlogger.log_level = :debug\n\nlogger.log(:info, 'MyApp', 'Application started - info')\nlogger.log(:debug, 'MyApp', 'Application started - debug')\nlogger.log(:error, 'MyApp', 'Application started - error')\nlogger.log(:fatal, 'MyApp', 'Application started - fatal')\nlogger.log(:warn, 'MyApp', 'Application started - warn')\n```\n\n### Utility helper functions, well tested\n\n*Q: The programming language does not provide basic functions in the standard library. I'm tempted to open my browser to add string manipulation and regular expression parser functions.*\n\n*A: Create a new file called `utility.{language-suffix}` and add a code comment on top. Instruct Code Suggestions to generate a string manipulation function ().*\n\nExample for a string manipulation method in Python:\n\n```python\n# Create a function to search for a string in a string, return the index\n# Optionally remove search string, and return modified string\n# Test the function on the string \"Cool programming languages: C++, Rust, Python\" - Remove the string \"C++\"\n\n# Potential AI-generated code below\ndef search_and_remove(text, search):\n    index = text.find(search)\n    if index != -1:\n        return index, text[:index] + text[index+len(search):]\n    else:\n        return None, text\n\ntext = \"Cool programming languages: C++, Rust, Python\"\nindex, new_text = search_and_remove(text, \"C++\")\nprint(index, new_text)\n```\n\n**Async exercise:** Ask GitLab Duo Chat how to add tests in Python, select the source code, and use the `/tests` slash command. \n\nA similar example can be implemented in Go, creating utility functions for unique integer values in an array, or the sum of all even values. Start with defining the project structure through `go mod init gitlab.com/group/project` and create the `array.go` file. Define the `package` and start with the first code comment to generate the functions.\n\n```go\npackage array \n\n// Create a function that returns unique values from an integer array\n\n// Possible AI-generated code\nfunc unique(ints []int) []int {\n    occurred := map[int]bool{}\n    result := []int{}\n    for _, i := range ints {\n        if !occurred[i] {\n            occurred[i] = true\n            result = append(result, i)\n        }\n    }\n    return result\n}\n\n// Create a function that returns the sum of all even numbers in an integer array\n\n// Possible AI-generated code\nfunc sumEvens(ints []int) int {\n    var sum int\n    for _, i := range ints {\n        if i%2 == 0 {\n            sum += i\n        }\n    }\n    return sum\n}\n```\n\n**Async exercise**: Create more utility helper functions in dedicated libraries, and use Chat to select and generate `/tests`. For the Go example, you can inspect potential solutions in the `go/utility/array_test.go` file in the [GitLab Duo Prompts project](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts). Build and test the code using `go build && go test`.\n\n### Generate regular expressions\n\nDevelopers' favorite one liners, never touched again. `git blame` knows very well but might not be able to provide enough context. GitLab Duo can help with regular expressions creation, explanation, and refactoring, in the following example:\n\n*Q: My regular expressions for parsing IPv6 and IPv4 addresses do not work. What's the best approach to solve this?*\n\n*A: Use Code Suggestions comments to generate examples using these regex types. Combine the questions with Chat, and ask for more examples in different languages. You can also select the existing source, and use a refined prompt with `/refactor using regular expressions` in the Chat prompt.*\n\n**Async exercise**: Choose your favorite language, create a function stub that checks IPv6 and IPv4 address strings for their valid format. Trigger Code Suggestions to generate a parsing regular expression code for you. Optionally, ask Chat how to refine and refactor the regex for greater performance.\n\nI chose TypeScript, a language on my personal learning list for 2024: `// Generate a TypeScript function which parses IPv6 and IPv4 address formats. Use regular expressions`.\n\n![Code Suggestions - typescript utility parse ip address regex](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_typescript_utility_parse_ip_address_regex.png)\n\n![Code Suggestions typescript - utility parse ip address regex tests](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_typescript_utility_parse_ip_address_regex_tests.png)\n\n## Re-trigger Code Suggestions\n\nYou can trigger Code Suggestions by pressing the `enter` or `space` key, depending on the context. In VS Code and the GitLab Web IDE, the GitLab Duo icon will appear in the same line, and at the bottom of the window.\n\nIf you accepted a suggestion, but actually want to try a different suggestion path, select the code, delete the line(s) and start over.\n\n> **Tip:** Different keystrokes and strategies for Code Suggestions are recorded in this video:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ORpRqp-A9hQ?si=CmA7PBJ9ckWsvjO3\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Common keyboard combinations to re-trigger Code Suggestions\n\nEspecially in the early adoption phase of Code Suggestions, you'll need to practice to get the best results from comments, existing code style, etc., put into context.\n\nA common keystroke pattern for triggering suggestions can be\n\n1. Press `Enter` and wait for the suggestion.\n1. Press `Space` followed by `Backspace` to immediately delete the whitespace again, or\n1. Press `Enter` to re-trigger the suggestion. `Backspace` to delete any leftover new lines.\n\nWhen a suggestion makes sense, or you want to see how far you can get:\n\n1. Continue pressing `Tab` to accept the suggestion.\n1. Add a space or press `Enter` to open a new scope for triggering a new suggestion.\n1. Continue accepting suggestions with `Tab`. \n\nNote that generative AI sometimes ends up in a loop of suggesting similar code paths over and over again. You can trigger this behavior by inserting test data into an array, using strings and numbers in a sorted order or by generating different API endpoints, as it tries to guess which other endpoints could be helpful. When this happens, break the acceptance flow, and continue writing code as normal.\n\n### Stuck in the middle of suggestions\n\nSometimes, the code suggestions may stop in the middle of a variable, function, etc. definition. If you are unsure about the syntax, or want to restart the code suggestions:\n\n1. Delete the last character(s) or the entire line, using `Backspace`.\n1. Alternatively, use `shift cursor left` (select characters) or `cmd shift cursor left` (select entire line), followed by `Backspace`.\n1. Move the cursor into the line above, and press `Enter` to force a Code Suggestions trigger again.\n\n### Code Suggestions stopped\n\nWhen Code Suggestions stops, there can be multiple reasons:\n\n1. The current file scope ends – for example, a `main()` function has been generated and closed.\n1. There could be connection problems to the GitLab instance (self-managed) or GitLab.com (SaaS, [Dedicated](https://about.gitlab.com/dedicated/)). Follow the [troubleshooting documentation](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/troubleshooting.html).\n\n## Code suggestions vs. code generation\n\nCode suggestions \"come as you go\" while writing code, and help with completing line(s). Code generation on the other hand requires more context to create entire code blocks, consisting of functions, algorithms, classes, etc. \n\nThe following sections discuss both methods, and how to get started with a practical example. \n\n### Code suggestions flow with comments\n\nUse your natural programming flow, and stop to adapt to adding code comments when helpful for context and better suggestions. You can accept code suggestions using the `Tab` key, or selectively accept words using the `cmd cursor right` keyboard shortcut.\n\nThe following new challenge implements a simple Linux statistics tool in C, mimicking the functionality of `iostat`, `vmstat` and `du` CLI commands on Linux. Sometimes, these low-level metrics come in handy for presenting application metrics, or otherwise help with debugging when requesting support data from customers.\n\nCreate a new application directory called `linux_stat` and `main.c` file, and let Code Suggestions complete everything else. First, start with a simple code comment block:\n\n```c\n// Create a Linux statistic tool\n// Collect IO, performance, disk usage, TCP latency\n// Print summaries\n```\n\nNext, press `Enter` to trigger the creation of the `main()` method. If not automatically created, force it through a new code comment.\n\n```c\n// Create main function to collect data\n```\n\nIn my attempts, generative AI did not immediately implement all functionality. Instead, it added a `TODO` comment into the following code block.\n\n```c\nint main() {\n    struct stat statbuf;\n\n    if (stat(\"/etc\", &statbuf) != 0) {\n        perror(\"stat\");\n        return 1;\n    }\n\n    printf(\"IO Block Size: %ld\\n\", statbuf.st_blksize);\n\n    // TODO: add CPU usage, disk usage, network latency measurements\n\n    return 0;\n}\n```\n\nMove the cursor after the `TODO` comment, and continue with a new comment to collect CPU usage:\n\n```c\n    // Collect CPU usage\n\n    // Example for AI-generated code below\n    double cpu_usage;\n\n    FILE* stat_file = fopen(\"/proc/stat\", \"r\");\n    if (stat_file == NULL) {\n        perror(\"fopen\");\n        return 1;\n    }\n\n    // Parse /proc/stat to get CPU usage\n    fscanf(stat_file, \"cpu %lf\", &cpu_usage);\n\n    printf(\"CPU Usage: %.2f%%\\n\", cpu_usage * 100);\n\n    fclose(stat_file);\n```\n\nThe generated code might not be 100% accurate, since measuring CPU usage on Linux requires taking samples over a defined time range. Thus, the caller needs to calculate the real CPU utilization from all values in `/proc/stat`, including [`system, nice, user, idle, iowait`, etc.](https://www.kernel.org/doc/Documentation/filesystems/proc.txt).\n\n**Async challenge:** Optimize the collection process with average calculation.\n\nRepeat the steps for disk usage and network latency. The code comment is written by you, the C source code is generated with GitLab Duo.\n\n```c\n    // Collect disk usage\n\n    // Collect network latency\n\n    // Example for AI-generated code below\n    int sockfd = socket(AF_INET, SOCK_STREAM, 0);\n\n    struct sockaddr_in servaddr;\n    servaddr.sin_family = AF_INET;\n    servaddr.sin_port = htons(80);\n    inet_pton(AF_INET, \"8.8.8.8\", &servaddr.sin_addr);\n```\n\nThe generated code may use public IP addresses to test network latency. Change the IPv4 address to GitLab.com's current address. \n\n```c\n    inet_pton(AF_INET, \"172.65.251.78\", &servaddr.sin_addr);\n```\n\nThe generated source code added more header dependencies. Navigate into the top section of the file, and add a comment to include all required headers:\n\n```c\n// Import required headers\n```\n\nBuild and run the application on Linux, example on Ubuntu 22 LTS:\n\n```shell\n# Install compiler\napt install gcc\n\n# Build\ngcc main.c -o linux_stat\n\n# Build: If the math.h header is included, linking against the library is needed, too\ngcc linux_stat.c -o linux_stat -lm\n\n# Run the tool\n./linux_stat\n```\n\n**Bonus:** Change the code to use IPv6, and check the GitLab.com IPv6 address again (`dig gitlab.com AAAA +short`).\n\n```c\n    // Collect network latency\n\n    // Use IPv6 \n\n    // Example for AI-generated code below\n    struct sockaddr_in6 servaddr;\n    servaddr.sin6_family = AF_INET6;\n    servaddr.sin6_port = htons(80);\n    //inet_pton(AF_INET6, \"2001:db8::1\", &servaddr.sin6_addr);\n    inet_pton(AF_INET6, \"2606:4700:90:0:f22e:fbec:5bed:a9b9\", &servaddr.sin6_addr);\n\n    int sockfd = socket(AF_INET6, SOCK_STREAM, 0);\n```\n\n![C Linux stat tests](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_c_linux_stat_tests.png)\n\nThe full working source code is available in the [GitLab Duo Prompts project](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts) in the directory for C code.\n\n**Async exercise:** Refactor the C code into Rust, using only GitLab Duo. Start by selecting the source code, and use the Duo Chat prompt `/refactor into Rust`. \n\n> **Tip:** Thoughtful code comments make the source code more readable, too. This helps new team members with onboarding, site reliability engineers with debugging production incidents, and open source contributors with [merging their first MRs](https://handbook.gitlab.com/handbook/marketing/developer-relations/contributor-success/community-contributors-workflows/#first-time-contributors).\n\n### Start with a comment on top for code generation\n\nSource code can be organized in multiple files. Whether you start with a new application architecture, or refactor existing source code, you can take advantage of code generation with GitLab Duo.\n\nStart with a comment block on top, and make it a step-by-step description. You can also break longer comments into multiple lines, revisiting the examples in this article. This pattern also helps to think about the requirements, and can help refining the prompts. \n\n```diff\n# Generate a webserver, using the Flask framework. \n# Implement the / URL endpoint with example output.\n+# Add an endpoint for Promtheus metrics\n\n// Create a database backend. \n// Abstract data handlers and SQL queries into function calls.\n+// Use PostgreSQL as default backend, and SQLite for developers as fallback.\n\n/* \nUse multi-threaded data access here.\nCreate a shared locked resource, and focus on supporting Linux pthreads. \n+Abstract the thread creation/wait procedures into object-oriented classes and methods.\n*/\n```\n\nMore code generation prompts for [supported programming languages](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html) are available in the [GitLab Duo Use Cases documentation](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html#code-generation-prompts).\n\n### Intent detection for code suggestions and generation\n\nCode Suggestions, depending on the GitLab Language Server in your IDE, will parse and detect the intent and offer code completion suggestions in the same line or code generation.\n\nThe technology in the background uses TreeSitter to parse the code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree), and determine whether the scope is inside a code comment block (generation), or inside the source code (completion). This detection needs to be executed fast on the client IDE, and proves to be a great use case for [WebAssembly](https://webassembly.org/). You can learn more in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/11568), and the following video, which provides a look into the GitLab Language Server powering Code Suggestions:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/VQlWz6GZhrs\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n### Tell a story for efficient code generation\n\nCode generation is art. Tell the story, and AI-powered GitLab Duo can assist you. \n\nThe following example aims to implement an in-memory key-value store in Go, similar to Redis. Start with a description comment, and trigger Code Suggestions by continuing with a new line and pressing `Enter`.\n\n```golang\n// Create an in-memory key value store, similar to Redis \n// Provide methods to\n// set/unset keys\n// update values\n// list/print with filters\n```\n\nWe can be more specific – which methods are required for data manipulation? Instruct Code Suggestions to generate methods for setting keys, updating values, and listing all contained data.\n\n```golang\n// Create an in-memory key value store, similar to Redis \n// Provide methods to\n// set/unset keys\n// update values\n// list/print with filters\n```\n\nAccept all suggestions using the `Tab` key. As a next step, instruct Code Suggestions to create a `main` function with test code.\n\n```golang\n// Create a main function and show how the code works\n```\n\nIf the test data is not enough, refine the generated code with a focus on extreme test cases.\n\n> **Tip:** You can use the same method for refined [Chat prompts and test generation](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#write-tests-in-the-ide), `/tests focus on extreme test cases`.\n\n```golang\n// Add more random test data, focus on extreme test cases\n```\n\n![Code Suggestions - go kv more test data](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_kv_more_test_data.png)\n\nThe full example, including fixed dependencies, is located in the [gitlab-duo-prompts project](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts) in the `code-suggestions/go/key-value-store` directory. Update the `main.go` file, and build and run the code using the following command: \n\n```shell\ngo build\n./key-value-store\n```\n\nThe first iteration was to create a standalone binary and test different implementation strategies for key-value stores. Commit the working code and continue with your GitLab Duo adoption journey in the next step.\n\n> **Tip:** New projects can benefit from Code Generation, and require practice and more advanced techniques to use code comments for prompt engineering. This method can also make experienced development workflows more efficient. Proof of concepts, new library introductions, or otherwise fresh iterations might not always be possible in the existing project and framework. Experienced developers seek to create temporary projects, and isolate or scope down the functionality. For example, introducing a database backend layer, and benchmarking it for production performance. Or, a library causing security vulnerabilities or license incompatibilities should be replaced with a different library, or embedded code functionality.\n\n### Iterate faster with code generation\n\nExperienced developers will say, \"There must be a key-value library in Go, let us not reinvent the wheel.\" Fortunately, Go is a mature language with a rich ecosystem, and awesome-go collection projects, for example [avelino/awesome-go](https://github.com/avelino/awesome-go), provide plenty of example libraries. Note: This possibility might not be the case for other programming languages, and requires a case-by-case review.\n\nWe can also ask GitLab Duo Chat first, `Which Go libraries can I use for key-value storage?`:\n\n![Chat - ask golang libraries kv](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_chat_ask_golang_libs_kv.png)\n\nAnd then refine the Code Suggestions prompt to specifically use the suggested libraries, for example, BoltDB.\n\n```diff\n// Create an in-memory key value store, similar to Redis \n// Provide methods to\n// set/unset keys\n// update values\n// list/print with filters\n+// Use BoltDB as external library\n```\n\nRepeat the pattern from above: Generate the source code functions, then ask GitLab Duo to create a main function with test data, and build the code. The main difference is external libraries, which need to be pulled with the `go get` command first. \n\n```shell\ngo get\ngo build\n```\n\nIf the source code build fails with missing dependencies such as `fmt`, practice using GitLab Duo again: Move the cursor into the `import` statement, and wait for the suggestion to add the missing dependencies. Alternatively, add a comment saying `Import all libraries`.\n\n![Code Suggestions - go kv external lib boltdb fix dependencies](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_kv_external_lib_boltdb_fix_deps.png)\n\nYou can also add more test data again, and verify how the functions behave: `// Add more random test data, focus on extreme test cases`. In the following example, an empty key causes the program to panic.\n\n![Code Suggestions - Go kv external lib boltdb test extreme cases panic](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_kv_external_lib_boltdb_test_extreme_cases_panic.png)\n\nThis example is a great preparation for test cases later on.\n\n### Practical code generation: Cloud-native observability\n\nThink of a client application in Go, which lists the current state of containers, pods, and services in a Kubernetes cluster, similar to the `kubectl get pods` command line. The Kubernetes project provides [Go libraries](https://pkg.go.dev/k8s.io/client-go/kubernetes) to programmatically interact with the Kubernetes APIs, interfaces, and object structures.\n\nOpen your IDE, and create a new Go project.\n\n> **Tip:** You can ask Chat how to do it - `How to start a Go project? Please show CLI command examples`. \n\nStart with a single comment on top of the `main.go` file, and describe the application purpose: Observability in Kubernetes.\n\n```golang\n// Create a client for Kubernetes observability\n```\n\nThink about the main requirements: Get access to Kubernetes, create context, namespace, and inspect the state. Additionally, instruct Code Suggestions to import packages and create a main package in the `main.go` file.\n\nFirst iteration:\n\n```golang\n// Create a client for Kubernetes observability\n// Inspect container, pod, service status and print an overview\n```\n\nThis might do unexpected things with hardcoding the access credentials, missing contexts, failing builds.\n\nSecond iteration:\n\n```golang\n// Create a client for Kubernetes observability\n// Create a function that\n// Read the kubernetes configuration file from the KUBECONFIG env var\n// Inspect container, pod, service status and print an overview\n```\n\nThis might not know about Kubernetes contexts and namespaces, thus leading to build errors or unexpected results.\n\nThird iteration:\n\n```golang\n// Create a client for Kubernetes observability\n// Create a function that\n// Read the kubernetes configuration file from the KUBECONFIG env var\n// Create kubernetes context, namespace default\n// Inspect container, pod, service status and print an overview\n```\n\nThis example hardcodes the Kubernetes context and default namespace to generate an initial foundation. Later iterations can read the namespace value from a command line parameter, or configuration file.\n\nThe final example can look like the following. In addition to the application functionality, it also instructs Code Suggestions to import all dependencies, and create a `main` package in `main.go`.\n\n```golang\n// Create a client for Kubernetes observability\n// Create a function that\n// Read the kubernetes configuration file from the KUBECONFIG env var\n// Create kubernetes context, namespace default\n// Inspect container, pod, service status and print an overview\n// Import necessary packages\n// Create main package\n```\n\n\u003Cdetails> Solution \n\u003Csummary>\n\n```golang\npackage main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nfunc main() {\n\tkubeconfig := os.Getenv(\"KUBECONFIG\")\n\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfig)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tclientset, err := kubernetes.NewForConfig(config)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tpods, err := clientset.CoreV1().Pods(\"\").List(context.TODO(), metav1.ListOptions{})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tfmt.Printf(\"There are %d pods in the cluster\\n\", len(pods.Items))\n\n\t// Additional code to inspect services, containers, etc\n}\n```\n\n\u003C/summary>\n\u003C/details>\n\nExample output:\n\n![duo code suggestions - go k8s o11y output](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_go_k8s_o11y_output.png)\n\n**Async exercise:** Complete the project with code for inspecting services, containers, etc., and export the findings to [OpenTelemetry](https://opentelemetry.io/).\n\n> **Tip:** Practice with the [GitLab Duo use cases: Code generation prompts](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html#code-generation-prompts) in the documentation, and/or send merge requests with your working prompts.\n\nWhile recording a short video to highlight how code generation is working, another more refined source code was generated. You can inspect the differences in [this commit](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts/-/commit/a1a46de9789d4791f04b4df9f1a35d05b8e67568), and benefit from both solutions.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/ORpRqp-A9hQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## Take advantage of all GitLab Duo features \n\n### Combine Chat with Code Suggestions\n\nIn combination with [GitLab Duo Chat](https://docs.gitlab.com/ee/user/gitlab_duo_chat/index.html), Code Suggestions becomes even more powerful. The following workflow illustrates the intersection of AI efficiency:\n\nWrite and generate new code using Code Suggestions. The source code will be verified through CI/CD automation, code quality tests, and security scanning. But what about the developer's knowledge?\n\n1. In your IDE, select the generated code portions and use the [`/explain` slash command](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#explain-code-in-the-ide) in the Chat prompt. You can even refine the prompt to `/explain with focus on algorithms`, or otherwise helpful scopes such as potential security or performance problems, etc.\n\n    - Continue writing and maintaining source code, but at some point code quality decreases and refactoring gets challenging. Ask GitLab Duo Chat for help.\n\n2. In your IDE, select the source code, and use the [`/refactor` slash command](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#refactor-code-in-the-ide) in the Chat prompt. You can refine the prompt to focus on specific design patterns (functions, object-oriented classes, etc.), `/refactor into testable functions` for example._\n\n    - After ensuring more readable code, tests need to be written. What are potential extreme cases, or random data examples for unit tests? Research and implementation in various frameworks can take time.\n\n3. In your IDE, select the source code, and use the [`/tests` slash command](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#write-tests-in-the-ide) in the Chat prompt. You can also refine the prompt to focus in specific test frameworks, scenarios, input methods, etc. \n\n    - Code quality and test coverage reports are green again. Focus on efficient DevSecOps workflows with Code Suggestions again. \n\nMore scenarios are described in the [GitLab Duo use cases documentation](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html).\n\n### Use Chat to generate build configuration\n\nThe time-intensive research on getting started with a new project can be exhausting. Especially with different paths to do it right, or alternative frameworks, this can lead to more work than anticipated. Newer programming languages like Rust propose one way (Cargo), while Java, C++, etc. offer multiple ways and additional configuration languages on top (Kotlin DSL, CMake DSL, etc.).\n\nTake advantage of asking GitLab Duo how to start a project, generate specific configuration examples for build tools (e.g. `Please show a gradle.build example for Spring Boot`), and reduce the time to start developing, building, and testing source code.\n\n1. Java, Gradle, Spring Boot: `Please show a gradle.build example for Spring Boot`\n1. C++, CMake, clang: `Please show a basic CMake configuration file for C++17, using clang as compiler.`\n1. Python: `Please show how to initialize and configure a Python project on the CLI`\n1. Rust: `Please show how to initialize and configure a Rust project.`, followed by a refinement question: `Explain the structure of Cargo.toml`.\n1. Go: `Please show how to initialize and configure a Go project`. \n\n### Use Chat to explain potential vulnerabilities\n\nLet us assume that some PHP code was generated to create a web form. The code might be vulnerable to security issues.\n\n```php\n\u003C?php \n// Create a feedback form for user name, email, and comments\n// Render a HTML form\n\n$name = $_POST['name'];\n$email = $_POST['email'];\n$comments = $_POST['comments'];\n\necho '\u003Cform method=\"post\">';\necho '\u003Clabel for=\"name\">Name:\u003C/label>';\necho '\u003Cinput type=\"text\" id=\"name\" name=\"name\">';\n\necho '\u003Clabel for=\"email\">Email:\u003C/label>';\necho '\u003Cinput type=\"email\" id=\"email\" name=\"email\">';\n\necho '\u003Clabel for=\"comments\">Comments:\u003C/label>';\necho '\u003Ctextarea id=\"comments\" name=\"comments\">\u003C/textarea>';\n\necho '\u003Cinput type=\"submit\" value=\"Submit\">'; \necho '\u003C/form>';\n\n?>\n```\n\nSelect the source code, and [ask Chat to explain](https://docs.gitlab.com/ee/user/gitlab_duo_chat/examples.html#explain-code-in-the-ide), using a refined prompt with `/explain why this code is vulnerable to bad security actors`. \n\n![Code Suggestions - Chat explains potential vulnerability](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_chat_explain_potential_vulnerability.png)\n\n> **Tip**: We are investigating and learning in the local developer environment. The vulnerable source code can be fixed before it reaches a Git push and merge request that trigger security scanning, which will unveil and track the problems, too. Learning about security vulnerabilities helps improve the developer experience.\n\n### Combine vulnerability resolution with Code Suggestions\n\nLets look into another example with an intentional [vulnerability resolution challenge](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/vulnerability-resolution/challenge-resolve-vulnerabilities), and see if we can use Code Suggestions in combination with vulnerability resolution. The linked project has been preconfigured with static application security testing (SAST) scanning. You can follow these steps to configure GitLab SAST by using the [SAST CI/CD component](https://gitlab.com/explore/catalog/components/sast) in the `.gitlab-ci.yml` CI/CD configuration file.\n\n```yaml\ninclude:\n  # Security: SAST (for vulnerability resolution)\n  - component: gitlab.com/components/sast/sast@1.1.0\n```\n\nAfter inspecting the vulnerability dashboard and details, you can use [vulnerability explanation](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#vulnerability-explanation) to better understand the context and potential problems. [Vulnerability resolution](https://docs.gitlab.com/ee/user/application_security/vulnerabilities/index.html#vulnerability-resolution) creates a rerge request with a proposed source code fix for a detected security vulnerability. \n\nSometimes, it can be necessary to refine the suggested code. Navigate into the [created MR](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-challenges/vulnerability-resolution/challenge-resolve-vulnerabilities/-/merge_requests/1), and either copy the Git branch path for local Git fetch, or open the Web IDE from the `Edit` button to continue in the browser. Navigate into the source code sections with the fixed code portions, and modify the code with a comment:\n\n```\n// refactor using safe buffers, null byte termination\n```\n\n![duo code suggestions - with vulnerability resolution proposal](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_with_vulnerability_resolution_proposal.png)\n\nAlternatively, you can also open Chat, select the source code and use the `/refactor` slash command.\n\n![duo code suggestions - with vulnerability resolution add duo chat refactor](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749677063/Blog/Content%20Images/duo_code_suggestions_with_vulnerability_resolution_add_duo_chat_refactor.png)\n\nA full example is available in the [GitLab Duo use cases documentation](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html#explain-and-resolve-vulnerabilities). \n\nHere is a recording of that example:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Ypwx4lFnHP0\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## More tips \n\n### Verify code quality and security\n\nMore generated code requires quality assurance, testing, and security measures. Benefit from all features on a DevSecOps platform:\n\n1. [CI/CD components](https://docs.gitlab.com/ee/ci/components/) and [pipeline efficiency](https://docs.gitlab.com/ee/ci/pipelines/pipeline_efficiency.html)\n1. [Code quality](https://docs.gitlab.com/ee/ci/testing/code_quality.html)\n1. [Code test coverage](https://docs.gitlab.com/ee/ci/testing/code_coverage.html)\n1. [Application security](https://docs.gitlab.com/ee/user/application_security/)\n1. [Observability](https://docs.gitlab.com/ee/operations/tracing.html)\n\n### Learn as a team, and understand AI impact\n\nAdapt and explore with dedicated team collaboration sessions, and record them for other teams to benefit from later. You can also follow the [GitLab Duo Coffee Chat playlist on YouTube](https://www.youtube.com/playlist?list=PL05JrBw4t0Kp5uj_JgQiSvHw1jQu0mSVZ).\n\nRead about AI impact metrics, including [How to put generative AI to work in your DevSecOps environment](https://about.gitlab.com/the-source/ai/how-to-put-generative-ai-to-work-in-your-devsecops-environment/) and the [Developing GitLab Duo: AI Impact analytics dashboard measures the ROI of AI](https://about.gitlab.com/blog/developing-gitlab-duo-ai-impact-analytics-dashboard-measures-the-roi-of-ai/). Visit the [AI Transparency Center](https://about.gitlab.com/ai-transparency-center/) to learn more about data usage, transparency, and AI ethics at GitLab.\n\n### Development is a marathon, not a sprint\n\nSometimes, code suggestions might take longer to load, compared to local auto-completion features. Take this time as an advantage, and think about the current algorithm or problem you are trying to solve. Often, a secondary thought can lead to more refined ideas. Or you can take a short break to take a sip from your preferred drink, and continue refreshed when the suggestions arrive.\n\nSome algorithms are super complex, or require code dependencies which cannot be resolved through auto-completion help. Proprietary and confidential code may provide less context to the large language models, and, therefore, require more context in the comments for Code Suggestions. Follow your own pace and strategy, and leverage Code Suggestions in situations where they help with boilerplate code, or helper functions. \n\n> **Tip:** Explore [Repository X-Ray](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/repository_xray.html) for more Code Suggestions context, and test experimental features, for example, [support for more languages in VS Code](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/supported_extensions.html#add-support-for-more-languages-for-code-suggestions-in-vs-code). More insights can be found in the epic to [improve acceptance rate for Code Suggestions](https://gitlab.com/groups/gitlab-org/-/epics/13085).\n\n### Contribute using GitLab Duo\n\nYou can use GitLab Duo to contribute to open source projects, using Code Suggestions, code refactoring, documentation through explanations, or test generation.\n\nGitLab customers can [co-create GitLab using GitLab Duo](https://docs.gitlab.com/ee/user/gitlab_duo/use_cases.html#use-gitlab-duo-to-contribute-to-gitlab), too. Follow the updated guidelines for [AI-generated contributions](https://about.gitlab.com/community/contribute/dco-cla/#ai-generated-contributions), and watch an example recording from the GitLab Duo Coffee Chat: Contribute to GitLab using Code Suggestions and Chat:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/TauP7soXj-E\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## Share your feedback\n\nGitLab Duo Code Suggestions enables more efficient development workflows. It requires hands-on practice and exercise through tutorials, team workshops, and guided training. Automated workflows with code quality, security scanning, and observability help tackle challenges with newly introduced source code at a much higher frequency. Taking advantage of all GitLab Duo features, including Chat, greatly improves the developer experience on the most comprehensive AI-powered DevSecOps platform.\n\nUse the best practices in this tutorial to kickstart your journey, follow the [GitLab Duo documentation](https://docs.gitlab.com/ee/user/gitlab_duo/index.html), and [ask our teams for GitLab Duo AI workshops](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/) (I have already shadowed customer workshops, they are great!). Please share your Code Suggestions feedback in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/435783), including screenshots and videos (when possible).\n\n> [Try GitLab Duo for free today!](https://about.gitlab.com/gitlab-duo/#free-trial)",[720,696,9],{"slug":4306,"featured":90,"template":700},"top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo","content:en-us:blog:top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo.yml","Top Tips For Efficient Ai Powered Code Suggestions With Gitlab Duo","en-us/blog/top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo.yml","en-us/blog/top-tips-for-efficient-ai-powered-code-suggestions-with-gitlab-duo",{"_path":4312,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4313,"content":4319,"config":4324,"_id":4326,"_type":13,"title":4327,"_source":15,"_file":4328,"_stem":4329,"_extension":18},"/en-us/blog/tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies",{"title":4314,"description":4315,"ogTitle":4314,"ogDescription":4315,"noIndex":6,"ogImage":4316,"ogUrl":4317,"ogSiteName":685,"ogType":686,"canonicalUrls":4317,"schema":4318},"Tutorial: Advanced use case for GitLab Pipeline Execution Policies","Learn how new GitLab Ultimate functionality can enforce a standardized pipeline across an organization for improved compliance.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098083/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_397632156_3Ldy1urjMStQCl4qnOBvE0_1750098083312.jpg","https://about.gitlab.com/blog/tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Advanced use case for GitLab Pipeline Execution Policies\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Dan Rabinovitz\"}],\n        \"datePublished\": \"2025-01-22\",\n      }",{"title":4314,"description":4315,"authors":4320,"heroImage":4316,"date":2326,"body":4322,"category":830,"tags":4323},[4321],"Dan Rabinovitz","[Pipeline execution\npolicies](https://docs.gitlab.com/ee/user/application_security/policies/pipeline_execution_policies.html)\nare a newer addition to the GitLab DevSecOps platform and a powerful\nmechanism to enforce CI/CD jobs across applicable projects. They enable\nplatform engineering or security teams to inject jobs into developers’ YAML\npipeline definition files, guaranteeing that certain CI/CD jobs will execute\nno matter what a developer defines in their \\`.gitlab-ci.yml\\` file. \n\n\nThis article will explain how to utilize pipeline execution policies to\ncreate guardrails around the stages or jobs that a developer can use in\ntheir pipeline definition. In regulated environments, this may be necessary\nto ensure developers adhere to a standard set of jobs or stages in their\nGitLab pipeline. Any job or stage that a developer adds to their pipeline\nthat does not adhere to a corporate standard will cause the pipeline to\nfail. \n\n\nOne example use case for pipeline execution policies is ensuring a security\nscanner job runs. Let’s say an organization has made an investment in a\nthird-party security scanner and they have a requirement that the external\nscan runs before any merge is made into the main branch. Without a pipeline\nexecution policy, a developer could easily skip this step by not including\nthe required code in their `.gitlab-ci.yml` file.  With a pipeline execution\npolicy in place, a security team can guarantee the external security\nscanning job executes regardless of how a developer defines their pipeline.\n\n\nTo use pipeline execution policies to enforce these restrictions requires\ntwo parts: a shell script to make calls to the GitLab API and the policy\nitself. This tutorial uses a bash script; if your runner uses a different\nscripting language, it is easy to adapt to other languages.\n\n\nHere is the example shell script I will use for this exercise:\n\n\n``` \n\n#!/bin/bash\n\n\necho \"Checking pipeline stages and jobs...\"\n\n\n# Pull the group access token from the environment variable\n\nGROUP_ACCESS_TOKEN=\"$PIPELINE_TOKEN\"\n\n\necho \"PROJECT_ID: $PROJECT_ID\"\n\necho \"PIPELINE_ID: $PIPELINE_ID\"\n\n\nif [ -z \"$GROUP_ACCESS_TOKEN\" ]; then  \n  echo \"GROUP_ACCESS_TOKEN (MR_GENERATOR) is not set\"\n  exit 1\nfi\n\n\nif [ -z \"$PROJECT_ID\" ]; then\n  echo \"PROJECT_ID is not set\"\n  exit 1\nfi\n\n\nif [ -z \"$PIPELINE_ID\" ]; then\n  echo \"PIPELINE_ID is not set\"\n  exit 1\nfi\n\n\n# Use the group access token for the API request\n\napi_url=\"$GITLAB_API_URL/projects/$PROJECT_ID/pipelines/$PIPELINE_ID/jobs\"\n\necho \"API URL: $api_url\"\n\n\n# Fetch pipeline jobs using the group access token\n\njobs=$(curl --silent --header \"PRIVATE-TOKEN: $GROUP_ACCESS_TOKEN\"\n\"$api_url\")\n\necho \"Fetched Jobs: $jobs\"\n\n\nif [[ \"$jobs\" == *\"404 Project Not Found\"* ]]; then\n  echo \"Failed to authenticate with GitLab API: Project not found\"\n  exit 1\nfi\n\n\n# Extract stages and jobs\n\npipeline_stages=$(echo \"$jobs\" | grep -o '\"stage\":\"[^\"]*\"' | cut -d '\"' -f 4\n| sort | uniq | tr '\\n' ',')\n\npipeline_jobs=$(echo \"$jobs\" | grep -o '\"name\":\"[^\"]*\"' | cut -d '\"' -f 4 |\nsort | uniq | tr '\\n' ',')\n\n\necho \"Pipeline Stages: $pipeline_stages\"  \n\necho \"Pipeline Jobs: $pipeline_jobs\"\n\n\n# Check if pipeline stages are approved\n\nfor stage in $(echo $pipeline_stages | tr ',' ' '); do \n  echo \"Checking stage: $stage\"\n  if ! [[ \",$APPROVED_STAGES,\" =~ \",$stage,\" ]]; then\n    echo \"Stage $stage is not approved.\"\n    exit 1\n  fi\ndone\n\n\n# Check if pipeline jobs are approved \n\nfor job in $(echo $pipeline_jobs | tr ',' ' '); do\n  echo \"Checking job: $job\"\n  if ! [[ \",$APPROVED_JOBS,\" =~ \",$job,\" ]]; then\n    echo \"Job $job is not approve\n```\n\n\nLet’s break this down a bit. \n\n\nThe first few lines of this code perform some sanity checks, ensuring that a\npipeline ID, project ID, and group access token exist.\n\n\n* A GitLab pipeline ID is a unique numerical identifier that GitLab\nautomatically assigns to each pipeline run.\n\n* A GitLab project ID is a unique numerical identifier assigned to each\nproject in GitLab.\n\n* A GitLab group access token is a token that authenticates and authorizes\naccess to resources at the group level in GitLab. This is in contrast to a\nGitLab personal access token (PAT), which is unique to each user.  \n\n\nThe bulk of the work comes from the [GitLab Projects\nAPI](https://docs.gitlab.com/ee/api/projects.html) call where the script\nrequests the jobs for the specified pipeline. Once you have job information\nfor the currently running pipeline, you can use a simple grep command to\nparse out stage and job names, and store them in variables for comparison.\nThe last portion of the script checks to see if pipeline stages and jobs are\non the approved list. Where do these parameters come from?\n\n\nThis is where [GitLab Pipeline Execution\nPolicies](https://docs.gitlab.com/ee/user/application_security/policies/pipeline_execution_policies.html)\ncome into play. They enable injection of YAML code into a pipeline. How can\nwe leverage injected YAML to execute this shell script?  Here’s a code\nsnippet showing how to do this.\n\n\n```\n\n## With this config, the goal is to create a pre-check job that evaluates\nthe pipeline and fails the job/pipeline if any checks do not pass\n\n\nvariables:\n  GITLAB_API_URL: \"https://gitlab.com/api/v4\"\n  PROJECT_ID: $CI_PROJECT_ID\n  PIPELINE_ID: $CI_PIPELINE_ID\n  APPROVED_STAGES: \".pipeline-policy-pre,pre_check,build,test,deploy\"\n  APPROVED_JOBS: \"pre_check,build_job,test_job,deploy_job\"\n\npre_check:\n  stage: .pipeline-policy-pre\n  script:\n    - curl -H \"PRIVATE-TOKEN:${REPO_ACCESS_TOKEN}\" --url \"https://\u003Cgitlab_URL>/api/v4/projects/\u003Cproject_id>/repository/files/check_settings.sh/raw\" -o pre-check.sh\n    - ls -l\n    - chmod +x pre-check.sh\n    - DEBUG_MODE=false ./pre-check.sh  # Set DEBUG_MODE to true or false\n  allow_failure: true\n```\n\n\nIn this YAML snippet, we set a few variables used in the shell script. Most\nimportantly, this is where approved stages and approved jobs are defined.\nAfter the `variables` section, we then add a new job to the\n`.pipeline-policy-pre` stage. This is a reserved stage for pipeline\nexecution policies and is guaranteed to execute before any stages defined in\na `.gitlab-ci.yml` file.  There is a corresponding `.pipeline-policy-post`\nstage as well, though we will not be using it in this scenario.  \n\n\nThe script portion of the job does the actual work. Here, we leverage a curl\ncommand to execute the shell script defined above. This example includes\nauthentication if it’s located in a private repository. However, if it’s\npublicly accessible, you can forgo this authentication. The last line\ncontrols whether or not the pipeline will fail. In this example, the\npipeline will continue. This is useful for testing – in practice, you would\nlikely set `allow_failure: false` to cause the pipeline to fail. This is\ndesired as the goal of this exercise is to not allow pipelines to continue\nexecution if a developer adds a rogue job or stage.\n\n\nTo utilize this YAML, save it to a `.yml` file in a repository of your\nchoice. We’ll see how to connect it to a policy shortly.\n\n\nNow, we have our script and our YAML to inject into a developer’s pipeline.\nNext, let’s see how to put this together using a pipeline execution policy.\n\n\nLike creating other policies in GitLab, start by creating a new Pipeline\nExecution Policy by navigating to **Secure > Policies** in the left hand\nnavigation menu. Then, choose **New Policy** at the top right, and select\n**Pipeline Execution Policy** from the policy creation options.  \n\n\nFor this exercise, you can leave the **Policy Scope** set to the default\noptions. In the **Actions** section, be sure to choose **Inject** and select\nthe project and file where you’ve saved your YAML code snippet. Click on\n**Update via Merge Request** at the very bottom to create an MR that you can\nthen merge into your project.\n\n\nIf this is your first security policy, clicking on **Merge** in the MR will\ncreate a [Security Policy\nProject](https://docs.gitlab.com/ee/user/application_security/policies/vulnerability_management_policy.html),\nwhich is a project to store all security policies. When implementing any\ntype of security policy in a production environment, [access to this project\nshould be restricted](https://docs.gitlab.com/ee/user/project/members/) so\ndevelopers cannot make changes to security policies. In fact, you may also\nwant to consider storing YAML code that’s used by pipeline execution\npolicies in this project to restrict access as well, though this is not a\nrequirement.  \n\nExecuting a pipeline where this pipeline execution policy is enabled should\nresult in the following output when you attempt to add an invalid stage to\nthe project `.gitlab-ci.yml` file.\n\n\n![Output of attempting an invalid stage to project gitlab-ci.yml\nfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098102/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098102394.png)\n\n\nWhile this use case is very focused on one aspect of security and compliance\nin your organization, this opens the door to other use cases. For example,\nyou may want to make group-level variables accessible to every project\nwithin a group; this is possible with pipeline execution policies. Or, you\nmay want to create a golden pipeline and have developers add to it. The\npossibilities are endless. GitLab customers are finding new and exciting\nways to use this new functionality every day.\n\n\nIf you’re a GitLab Ultimate customer, try this out today and let us know how\nyou’re using pipeline execution policies. Not a GitLab Ultimate customer?\n[Sign up for a free\ntrial](https://about.gitlab.com/free-trial/devsecops/) to get started.\n\n\n## Read more\n\n- [How to integrate custom security scanners into\nGitLab](https://about.gitlab.com/blog/how-to-integrate-custom-security-scanners-into-gitlab/)\n\n- [Integrate external security scanners into your DevSecOps\nworkflow](https://about.gitlab.com/blog/integrate-external-security-scanners-into-your-devsecops-workflow/)\n\n- [Why GitLab is deprecating compliance pipelines in favor of security\npolicies](https://about.gitlab.com/blog/why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies/)\n",[830,9,185,495,108,696],{"slug":4325,"featured":6,"template":700},"tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies","content:en-us:blog:tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies.yml","Tutorial Advanced Use Case For Gitlab Pipeline Execution Policies","en-us/blog/tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies.yml","en-us/blog/tutorial-advanced-use-case-for-gitlab-pipeline-execution-policies",{"_path":4331,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4332,"content":4338,"config":4343,"_id":4345,"_type":13,"title":4346,"_source":15,"_file":4347,"_stem":4348,"_extension":18},"/en-us/blog/tutorial-automated-release-and-release-notes-with-gitlab",{"title":4333,"description":4334,"ogTitle":4333,"ogDescription":4334,"noIndex":6,"ogImage":4335,"ogUrl":4336,"ogSiteName":685,"ogType":686,"canonicalUrls":4336,"schema":4337},"Tutorial: Automate releases and release notes with GitLab","With the GitLab Changelog API, you can automate the generation of release artifacts, release notes, and a comprehensive changelog detailing all user-centric software modifications.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749659978/Blog/Hero%20Images/automation.png","https://about.gitlab.com/blog/tutorial-automated-release-and-release-notes-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Automate releases and release notes with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ben Ridley\"}],\n        \"datePublished\": \"2023-11-01\",\n      }",{"title":4333,"description":4334,"authors":4339,"heroImage":4335,"date":3146,"body":4341,"category":694,"tags":4342,"updatedDate":692},[4340],"Ben Ridley","***2025 update** - The Changelog API has continued to evolve and now has some great new capabilities we don’t cover in this blog, such as the ability to provide custom changelogs with templated values from your commit history. [Discover more in the official Changelogs docs.](https://docs.gitlab.com/user/project/changelogs/)*\n\nWhen you develop software that users rely on, effective communication about changes with each release is essential. By keeping users informed about new features and any modifications or removals, you ensure they maximize the software's benefits and avoid encountering unpleasant surprises during upgrades.\n\nHistorically, creating release notes and maintaining a changelog has been a laborious task, requiring developers to monitor changes externally or release managers to sift through merge histories. With the GitLab Changelog API, you can use the rich history provided in our git repository to easily create release notes and maintain a changelog.\n\nIn this tutorial, we'll delve into automating releases with GitLab, covering the generation of release artifacts, release notes, and a comprehensive changelog detailing all user-centric software modifications.\n\n## Releases in GitLab\nFirst, let's explore how releases work in GitLab.\n\nIn GitLab, a release is a specific version of your code, identified by a git tag, that includes details about changes since the last release (and release notes) and any related artifacts built from that version of the code, such as Docker images, installation packages, and documentation.\n\nYou can create and track releases in GitLab using the UI by calling our Release API or by defining a special `release` job inside a CI pipeline. In this tutorial, we'll use the `release` job in a CI/CD pipeline, which allows us to extend the automation we're using in our pipelines for testing, code scanning, etc. to also perform automated releases.\n\nTo automate our releases, we first need to answer this question: Where are we going to get the information on changes made for our release notes and our changelog? The answer: Our git repository, which provides us with a rich history of development activity through commit messages and merge commit history. Let's see if we can leverage this rich history to automatically create our notes and changelogs.\n\n## Introducing commit trailers\n[Commit trailers](https://git-scm.com/docs/git-interpret-trailers) are structured entries in your git commits, created by adding simple `\u003CHEADER>:\u003CBODY>` format messages to the end of your commit. The `git` CLI tool can then parse and extract these for use in other systems. An example you might have already used is `git commit --sign-off` to sign off on a commit. This is implemented by adding a `Signed-off-by: \u003CYour Name>` trailer to the commit. We can add any arbitrary structured data here, which makes it a great place to store information that could be useful for our changelog.\n\nIn fact, if we use a `Changelog: \u003Cadded/changed/removed>` trailer in our commits, the GitLab Changelog API will parse these and use them to create a changelog for us automatically!\n\nLet's see this in action by making some changes to a real codebase and performing a release, and generating release notes and changelog entries.\n\n## Our example project\nFor the purposes of this blog, I'm using a simple Python web app repository. Let's pretend Version 1.0.0 of the application was just released and is the current version of the code. I've also created a 1.0.0 release in GitLab, which I did manually because we haven't created our automated release pipeline yet:\n\n![A screenshot of the GitLab UI showing a release for Version 1.0.0](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/1-0-release.png)\n\n## Making our changes\nWe're in rapid development mode, so we're going to be working on releasing Version 2.0.0 of our application today. As part of our 2.0.0 release, we're going to be adding a new feature to our app: A chatbot! And we're also going to be removing the quantum blockchain feature, because we only needed that for our first venture capital funding round. Also, we're going to be adding an automated release job to our CI/CD pipeline for our 2.0.0 release.\n\nFirst, let's remove unneeded features. I've created a merge request that contains the necessary removals. Importantly, we need to ensure we have a commit message that includes the `Changelog: removed` trailer. There's a few ways to do this, such as including it directly in a commit, or performing an interactive rebase and adding it using the CLI. But I think the easiest way in our situation is to leave it until the end and then use the `Edit commit message` button in GitLab to add the trailer to the merge commit like so:\n\n![A screenshot the GitLab UI showing a merge request removing unused features](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/remove-unused-features-mr.png)\n\nIf you use this method, you can also change the merge commit title to something more succinct. I've changed the title of my merge commit to 'Remove Unused Features', as this is what will appear in the changelog entry.\n\nNext, let's add some new functionality for the 2.0.0 release. Again, all we need to do is open another merge request that includes our new features and then edit the merge commit to include the `Changelog: added` trailer and edit the commit title to be more succinct:\n\n![A screenshot of the GitLab UI showing a merge request to add new functionality](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/add-chatbot-mr.png)\n\nNow we're pretty much ready to release 2.0.0. But we don't want to create our release manually this time. So before our release we're going to add some jobs to our `.gitlab-ci.yml` file that will perform the release for us automatically, and generate the respective release notes and changelog entries, when we tag our code with a new version like `2.0.0`.\n\n**Note:** If you want to enforce changelog trailers, consider using something like [Danger to perform automated checks for MR conventions](https://docs.gitlab.com/ee/development/dangerbot.html).\n\n## Building an automated release pipeline\nFor our pipeline to work, we need to create a project access token that will allow us to call GitLab's API to generate changelog entries. [Create a project access token with the API scope](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#create-a-project-access-token), and then [store the token as a CI/CD variable](https://docs.gitlab.com/ee/ci/variables/#define-a-cicd-variable-in-the-ui) called `CI_API_TOKEN`. We'll reference this variable to authenticate to the API.\n\nNext, we're going to add two new jobs to our `gitlab-ci.yml` file:\n```yaml\nprepare_job:\n  stage: prepare\n  image: alpine:latest\n  rules:\n  - if: '$CI_COMMIT_TAG =~ /^v?\\d+\\.\\d+\\.\\d+$/'\n  script:\n    - apk add curl jq\n    - 'curl -H \"PRIVATE-TOKEN: $CI_API_TOKEN\" \"$CI_API_V4_URL/projects/$CI_PROJECT_ID/repository/changelog?version=$CI_COMMIT_TAG\" | jq -r .notes > release_notes.md'\n  artifacts:\n    paths:\n    - release_notes.md\n\nrelease_job:\n  stage: release\n  image: registry.gitlab.com/gitlab-org/release-cli:latest\n  needs:\n    - job: prepare_job\n      artifacts: true\n  rules:\n  - if: '$CI_COMMIT_TAG =~ /^v?\\d+\\.\\d+\\.\\d+$/'\n  script:\n    - echo \"Creating release\"\n  release:\n    name: 'Release $CI_COMMIT_TAG'\n    description: release_notes.md\n    tag_name: '$CI_COMMIT_TAG'\n    ref: '$CI_COMMIT_SHA'\n    assets:\n      links:\n        - name: 'Container Image $CI_COMMIT_TAG'\n          url: \"https://$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA\"\n```\n\nIn the above configuration, the `prepare_job` uses `curl` and `jq` to call the GitLab Changelog API endpoint and then passes this to our `release_job` to actually create the release. To break it down further:\n- We use the project access token created earlier to call the GitLab Changelog API, which performs the generation of the release notes and we store this as an artifact.\n- We're using the `$CI_COMMIT_TAG` variable as the version. For this to work, we need to be using semantic versioning for our tags (something like `2.0.0` for example), so you'll notice I've also restricted the release job using a `rules` section that checks for a semantic version tag.\n\t- Semantic versioning is required for the GitLab Changelog API to work. It uses this format to find the most recent release to compare to our current release.\n- We use the official `release-cli` image from GitLab. The release-cli is required to use the `release` keyword in a job.\n- We use the `release` keyword to create a release in GitLab. This is a special job keyword reserved for creating a release and populating the required fields.\n- We can pass a file as an argument to the `description` of the release. In our case, it's the file we generated in the `prepare_job`, which was passed to this job as an artifact.\n- We've also included our container image that is being built earlier in the pipeline as a release asset. You can attach any assets you'd like from your build process, such as binaries or documentation by providing a URL to wherever you've uploaded them earlier in the pipeline.\n\n## Performing an automated release\nWith this setup, all we need to do to perform a release is push a tag to our repository that follows our versioning scheme. You can simply push a tag using the CLI, this example uses GitLab's UI to create a tag on the main branch. Create a tag by selecting Code -> Tags -> New Tag on the sidebar:\n![A screenshot of the GitLab UI illustrating how to create a tag](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/create-2-tag.png)\n\nOn creation, our pipelines will start to execute. The GitLab Changelog API will automatically generate release notes for us as markdown, which contains all the changes between this release and the previous release. Here's the resulting markdown generated in our example:\n\n```md\n## 2.0.0 (2023-08-25)\n\n### added (1 change)\n\n- [Add ChatBot](gl-demo-ultimate-bridley/super-devsecops-incorporated/simply-notes-release-demo@0c3601a45af617c5481322bfce4d71db1f911b02) ([merge request](gl-demo-ultimate-bridley/super-devsecops-incorporated/simply-notes-release-demo!4))\n\n### removed (1 change)\n\n- [Remove Unused Features](gl-demo-ultimate-bridley/super-devsecops-incorporated/simply-notes-release-demo@463d453c5ae0f4fc611ea969e5442e3298bf0d8a) ([merge request](gl-demo-ultimate-bridley/super-devsecops-incorporated/simply-notes-release-demo!3))\n```\n\nAs you can see, GitLab has extracted the entries for our release notes automatically using our git commit trailers. In addition, it's helpfully provided links back to the merge request so readers can see more details and discussion around the changes.\n\nAnd now, our final release:\n![The GitLab release UI showing a release for version 2.0.0](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/2-0-release.png)\n\n## Creating the changelog\nNext, we want to update our changelog (which is basically a collated history of all your release notes). You can use a `POST` request to the changelog API endpoint we used earlier to do this.\n\nYou can do this as part of your release pipeline if you like, for example by adding this to the `script` section of your prepare job:\n```sh\n'curl -H \"PRIVATE-TOKEN: $CI_API_TOKEN\" -X POST \"$CI_API_V4_URL/projects/$CI_PROJECT_ID/repository/changelog?version=$CI_COMMIT_TAG\"\n```\n\n**Note that this will actually modify the repository.** It will create a commit to add the latest notes to a `CHANGELOG.md` file:\n![A screenshot of the repository which shows a commit updating the changelog file](https://about.gitlab.com/images/blogimages/2023-08-22-automated-release-and-release-notes-with-gitlab/changelog-api-commit.png)\n\nAnd we are done! By utilizing the rich history provided by `git` with some handy commit trailers, we can leverage GitLab's powerful API and CI/CD pipelines to automate our release process and generate release notes for us.\n\n> If you’d like to explore the project we used for this article, [you can find the project at this link](https://gitlab.com/gitlab-learn-labs/sample-projects/release-automation-demo).\n",[9,787,108,789,851,742],{"slug":4344,"featured":6,"template":700},"tutorial-automated-release-and-release-notes-with-gitlab","content:en-us:blog:tutorial-automated-release-and-release-notes-with-gitlab.yml","Tutorial Automated Release And Release Notes With Gitlab","en-us/blog/tutorial-automated-release-and-release-notes-with-gitlab.yml","en-us/blog/tutorial-automated-release-and-release-notes-with-gitlab",{"_path":4350,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4351,"content":4357,"config":4362,"_id":4364,"_type":13,"title":4365,"_source":15,"_file":4366,"_stem":4367,"_extension":18},"/en-us/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component",{"title":4352,"description":4353,"ogTitle":4352,"ogDescription":4353,"noIndex":6,"ogImage":4354,"ogUrl":4355,"ogSiteName":685,"ogType":686,"canonicalUrls":4355,"schema":4356},"Tutorial: How to set up your first GitLab CI/CD component","Use Python scripts in your GitLab CI/CD pipelines to improve usability. In this step-by-step guide, you'll learn how to get started building your own CI/CD component.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098410/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2826%29_3lH4gZFVIGCndksN6Rlg85_1750098409928.png","https://about.gitlab.com/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: How to set up your first GitLab CI/CD component\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sophia Manicor\"},{\"@type\":\"Person\",\"name\":\"Noah Ing\"}],\n        \"datePublished\": \"2024-11-12\",\n      }",{"title":4352,"description":4353,"authors":4358,"heroImage":4354,"date":4359,"body":4360,"category":785,"tags":4361},[2345,847],"2024-11-12","Do you use Python scripts in your GitLab CI pipelines? Do you want to create\npipelines at scale? This tutorial shows how to set up your first [GitLab\nCI/CD component](https://docs.gitlab.com/ee/ci/components/) to deploy Python\nscripts. \n\n\nA [CI/CD component is a reusable single pipeline configuration\nunit](https://about.gitlab.com/blog/introducing-ci-components/).\nUse components to create a small part of a larger pipeline, or even to\ncompose a complete pipeline configuration.\n\n\n# Prerequisites\n\n- Basic Python knowledge\n\n- Working knowledge of GitLab CI\n\n- 8 minutes\n\n\n## Python script \n\n\n* **[The demo Python\nscript](https://gitlab.com/demos/templates/gitlab-python-cicd-component/-/blob/main/src/script.py?ref_type=heads)**\n\n\nThis Python script utilizes a library called\n[ArgParse](https://docs.python.org/3/library/argparse.html) . ArgParse\nallows you to pass variables to script through the command line. This script\ntakes in three arguments:\n\n[Python_container_image](https://docs.gitlab.com/ee/ci/yaml/#image): This is\nthe Python container image you wish to use.\n\n[Stage](https://docs.gitlab.com/ee/ci/yaml/#stage): This is the GitLab CI\nstage in which you job will run in. \n\nName: This is your name.\n\n\n```python\n\nimport argparse\n\n\nparser = argparse.ArgumentParser(description='Python CICD Component\nBoilerplate')\n\nparser.add_argument('python_container_image', type=str,\nhelp='python:3.10-slim')\n\nparser.add_argument('stage', type=str, help='Build')\n\nparser.add_argument('persons_name', type=str, help='Noah')\n\nargs = parser.parse_args()\n\n\npython_container_image = args.python_container_image\n\nstage = args.stage\n\npersons_name = args.persons_name\n\n```\n\n\nThis will take in these three variables and print out simple statements:\n\n\n```python\n\nprint(\"You have chosen \" + python_container_image + \" as the container\nimage\")\n\nprint(\"You have chosen \" + stage + \" as the stage to run this job\")\n\nprint(\"Thank you \" + persons_name + \"! you are succesfully using GitLab CI\nwith a Python script.\")\n\n```\n\n\nTo test this script locally, you can call on the script by utilizing the\nfollowing command:\n\n\n```bash\n\npython3 src/script.py python_container_image stage name\n\n```\n\n\nModify this script accordingly if you’d like to add in your own arguments!\n\n\n## Template \n\n\n* **[Demo of\ntemplate](https://gitlab.com/demos/templates/gitlab-python-cicd-component/-/blob/main/templates/template.yml?ref_type=heads)**\n\n\n**Note:** As long as the `gitlab-ci.yml` is placed in the\ntemplates/directory, the CI/CD component will know to pick it up. We named\nour template `templates.yml`, but any name would work for this YAML file.\n\n\nNow, getting into the fun part of CI/CD components, inputs! \n[Inputs](https://docs.gitlab.com/ee/ci/yaml/inputs.html) allow you to pass\nthrough variables into your pipeline. \n\n\n```yml\n\nspec:\n  inputs:\n    python_container_image:\n      default: python:3.10-slim\n      description: \"Define any python container image\"\n    stage:\n      default: build\n      description: \"Define the stage this job will run in\"\n    persons_name:\n      default: Noah\n      description: \"Put your name here\"\n```\n\nHere we have defined the three inputs that are our arguments in our Python\nscript. You can see for each input we have added in a default value – this\nwill be what the input is set to if not overridden. If we took out this\ndefault keyword the input would become mandatory when we use our component.\nAs it is written now, adding in these inputs when we use our component is\noptional due to our default values.\n\n\nWe can also set descriptions to ensure that other developers can understand\nwhat to input when they use our component. Descriptions are optional but\nthey provide self documentation within the code itself, which is always\nnice.\n\n\nAfter we set up our inputs, let’s write the rest of our component:\n\n\n```yml\n\ncomponent:\n  image: $[[ inputs.python_container_image ]]\n  stage: $[[ inputs.stage ]]\n  before_script:\n    - pip3 install -r src/requirements.txt\n  script: python3 src/script.py $[[ inputs.python_container_image ]] $[[ inputs.stage ]] $[[ inputs.persons_name ]]\n```\n\n\nTo use inputs in our component, we need to use the syntax `$[[\ninputs.$VARIABLE ]]`. In the above code, you can see that we use inputs to\ndefine our image and stage with  `$[[ inputs.python_container_image ]]`\nand   `$[[ inputs.stage ]] `.\n\n\n```\n\nscript: python3 src/script.py $[[ inputs.python_container_image ]] $[[\ninputs.stage ]] $[[ inputs.persons_name ]]\n\n```\n\nDiving into the script section, you can see we call upon our Python script..\nWe are able to pass our inputs in with the help of the ArgParse.\n\n\nNow that you have reviewed how the Python script works and the template has\nbeen set up, it is time to use the component!\n\n\n## Using the component \n\n\n* **[A demo of including the\ncomponent](https://gitlab.com/demos/templates/gitlab-python-cicd-component/-/blob/main/.gitlab-ci.yml?ref_type=heads)\n\n\nIn order to utilize the CI/CD component we just created, we need to include\nit in the `.gitlab-ci.yml` file that is in the root of our directory. \n\n\n```\n\ninclude:\n  # include the component located in the current project from the current SHA\n  - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/template@$CI_COMMIT_SHA\n    inputs:\n      python_container_image: python:3.11-slim\n      stage: test\n      persons_name: Tanuki\n```\n\n\nOne way to include it is to call upon it locally in the current project from\nthe current `Commit SHA`. You can find other ways to [reference a component\nin our\ndocumentation](https://docs.gitlab.com/ee/ci/components/#use-a-component).\n\n\nTo override the defaults, we have passed in other inputs so we get the\ncorrect image, stage, and name for our job. \n\n\nTry and change the `persons_names` to your own and watch the pipeline run!\n\n\n![ci/cd component tutorial - pipeline\nrunning](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098419/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098418901.png)\n\n\nVoila! You have learned how to set up a basic C/ICD component utilizing a\nPython ArgParse script!\n\n\n## What's next?\n\nIn the Python script, there is a commented out GitLab Python library and OS\nlibrary. If you would like to interact with the GitLab API, you can\nuncomment these and add in a [GitLab personal access\ntoken](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)\nto the [CI/CD variables](https://docs.gitlab.com/ee/ci/variables/) named\n`GLPAT`.\n\n\n```\n\nimport gitlab\n\nimport os\n\n```\n\nAfterwards you can then interact with the GitLab API.\n\n\n```\n\nglpat = os.environ['GLPAT']\n\n\ngl = gitlab.Gitlab(private_token=glpat)\n\n# SELF_HOSTED gl = gitlab.Gitlab(url='https://gitlab.example.com',\nprivate_token='xxxxxxxxxxxxxx')\n\ntry:\n   projects = gl.projects.list(get_all=True)\n   print(projects)\nexcept Exception as error:\n   print(\"Error:\", error)\n```\n\n\n> Learn more about CI/CD components and how to avoid building pipelines from\nscratch with the [GitLab CI/CD\nCatalog](https://about.gitlab.com/blog/ci-cd-catalog-goes-ga-no-more-building-pipelines-from-scratch/). \n\n\n## Read more\n\n\n- [FAQ: GitLab CI/CD\nCatalog](https://about.gitlab.com/blog/faq-gitlab-ci-cd-catalog/)\n\n- [Introducing CI/CD Steps, a programming language for DevSecOps\nautomation](https://about.gitlab.com/blog/introducing-ci-cd-steps-a-programming-language-for-devsecops-automation/)\n\n- [A CI/CD component builder's\njourney](https://about.gitlab.com/blog/a-ci-component-builders-journey/)\n",[108,9,851,1016],{"slug":4363,"featured":90,"template":700},"tutorial-how-to-set-up-your-first-gitlab-ci-cd-component","content:en-us:blog:tutorial-how-to-set-up-your-first-gitlab-ci-cd-component.yml","Tutorial How To Set Up Your First Gitlab Ci Cd Component","en-us/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component.yml","en-us/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component",{"_path":4369,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4370,"content":4375,"config":4380,"_id":4382,"_type":13,"title":4383,"_source":15,"_file":4384,"_stem":4385,"_extension":18},"/en-us/blog/tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access",{"title":4371,"description":4372,"ogTitle":4371,"ogDescription":4372,"noIndex":6,"ogImage":1875,"ogUrl":4373,"ogSiteName":685,"ogType":686,"canonicalUrls":4373,"schema":4374},"Tutorial: Install VS Code on a cloud provider VM and set up remote access","Learn how to automate the installation of VS Code on a VM running on a cloud provider and how to access it from your local laptop.","https://about.gitlab.com/blog/tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Install VS Code on a cloud provider VM and set up remote access\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2024-05-06\",\n      }",{"title":4371,"description":4372,"authors":4376,"heroImage":1875,"date":4377,"body":4378,"category":785,"tags":4379},[715],"2024-05-06","DevSecOps teams can sometimes find they need to run an instance of Visual\nStudio Code (VS Code) remotely for team members to share when they don't\nhave enough local resources. However, installing, running, and using VS Code\non a remote virtual machine (VM) via a cloud provider can be a complex\nprocess full of pitfalls and false starts. This tutorial covers how to\nautomate the installation of VS Code on a VM running on a cloud provider.\n\n\nThis approach involves two separate GitLab projects, each with its own\npipeline. The first one uses Terraform to instantiate a virtual machine in\nGCP running Linux Debian. The second one installs VS Code on the newly\ninstantiated VM. Lastly, we provide a procedure on how to set up your local\nMac laptop to connect and use the VS Code instance installed on the remote\nVM.\n\n\n## Create a Debian Linux distribution VM on GCP\n\n\nHere are the steps to create a Debian Linux distribution VM on GCP.\n\n\n### Prerequisites\n\n\n1. A GCP account. If you don't have one, please [create\none](https://cloud.google.com/free?hl=en).\n\n2. A GitLab account on [gitlab.com](https://gitlab.com/users/sign_in)\n\n\n**Note:** This installation uses:\n\n\n- Debian 5.10.205-2 (2023-12-31) x86_64 GNU/Linux, a.k.a Debian 11\n\n\n### Create a service account and download its key\n\n\nBefore you create the first GitLab project, you need to create a service\naccount in GCP and then generate and download a key. You will need this key\nso that your GitLab pipelines can communicate to GCP and the GitLab API.\n\n\n1. To authenticate GCP with GitLab, sign in to your GCP account and create a\n[GCP service\naccount](https://cloud.google.com/docs/authentication#service-accounts) with\nthe following roles:\n\n- `Compute Network Admin`\n\n- `Compute Admin`\n\n- `Service Account User`\n\n- `Service Account Admin`\n\n- `Security Admin`\n\n\n3. Download the JSON file with the service account key you created in the\nprevious step.\n\n4. On your computer, encode the JSON file to `base64` (replace\n`/path/to/sa-key.json` to the path where your key is located):\n\n   ```shell\n   base64 -i /path/to/sa-key.json | tr -d \\\\n\n   ```\n\n**NOTE:** Save the output of this command. You will use it later as the\nvalue for the `BASE64_GOOGLE_CREDENTIALS` environment variable.\n\n\n### Configure your GitLab project\n\n\nNext, you need to create and configure the first GitLab project.\n\n\n1. Create a group in your GitLab workspace and name it `gcpvmlinuxvscode`.\n\n\n1. Inside your newly created group, clone the following project:\n\n   ```shell\n   git@gitlab.com:tech-marketing/sandbox/gcpvmlinuxvscode/gcpvmlnxsetup.git\n   ```\n\n1. Drill into your newly cloned project, `gcpvmlnxsetup`, and set up the\nfollowing CI/CD variables to configure it:\n   1. On the left sidebar, select **Settings > CI/CD**.\n   1. Expand **Variables**.\n   1. Set the variable `BASE64_GOOGLE_CREDENTIALS` to the `base64` encoded JSON file you created in the previous section.\n   1. Set the variable `TF_VAR_gcp_project` to your GCP `project` ID.\n   1. Set the variable `TF_VAR_gcp_region` to your GCP `region` ID, e.g. us-east1, which is also its default value.\n   1. Set the variable `TF_VAR_gcp_zone` to your GCP `zone` ID, e.g. us-east1-d, which is also its default value.\n   1. Set the variable `TF_VAR_machine_type` to the GCP `machine type` ID, e.g. e2-standard-2, which is also its default value.\n   1. Set the variable `TF_VAR_gcp_vmname` to the GCP `vm name` you want to give the VM, e.g. my-test-vm, which is also its default value.\n\n**Note:** We have followed a minimalist approach to set up this VM. If you\nwould like to customize the VM further, please refer to the [Google\nTerraform\nprovider](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference)\nand the [Google Compute Instance Terraform\nprovider](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance)\ndocumentation for additional resource options.\n\n\n### Provision your VM\n\n\nAfter configuring your project, manually trigger the provisioning of your VM\nas follows:\n\n\n1. On the left sidebar, go to **Build > Pipelines**.\n\n1. Next to **Play** (**{play}**), select the dropdown list icon\n(**{chevron-lg-down}**).\n\n1. Select **Deploy** to manually trigger the deployment job.\n\n\nWhen the pipeline finishes successfully, you can see your new VM on GCP:\n\n\n- Check it on your [GCP console's VM instances\nlist](https://console.cloud.google.com/compute/instances).\n\n\n### Remove the VM\n\n\n**Important note:** Only run the cleanup job when you no longer need the GCP\nVM and/or the VS Code that you installed in it.\n\n\nA manual cleanup job is included in your pipeline by default. To remove all\ncreated resources:\n\n\n1. On the left sidebar, select **Build > Pipelines** and select the most\nrecent pipeline.\n\n1. For the `destroy` job, select **Play** (**{play}**).\n\n\n## Install and set up VS Code on a GCP VM\n\n\nPerform the steps in this section only after you have successfully finished\nthe previous sections above. In this section, you will create the second\nGitLab project that will install VS Code and its dependencies on the running\nVM on GCP.\n\n\n### Prerequisites\n\n\n1. A provisioned GCP VM. We covered this in the previous sections.\n\n\n**Note:** This installation uses:\n\n\n- VS Code Version 1.85.2\n\n\n### Configure your project\n\n\n**Note:** Since you will be using the `ssh` command multiple times on your\nlaptop, we strongly suggest that you make a backup copy of your laptop local\ndirectory `$HOME/.ssh` before continuing.\n\n\nNext, you need to create and configure the second GitLab project.\n\n\n1. Head over to your GitLab group `gcpvmlinuxvscode`, which you created at\nthe beginning of this post.\n\n\n1. Inside group, `gcpvmlinuxvscode`, clone the following project:\n\n   ```shell\n   git@gitlab.com:tech-marketing/sandbox/gcpvmlinuxvscode/vscvmsetup.git\n   ```\n\n1. Drill into your newly cloned project, `vscvmsetup` and set up the\nfollowing CI/CD variables to configure it:\n   1. On the left sidebar, select **Settings > CI/CD**.\n   1. Expand **Variables**.\n   1. Set the variable `BASE64_GOOGLE_CREDENTIALS` to the `base64` encoded JSON file you created in project `gcpvmlnxvsc`. You can copy this value from the variable with the same name in project `gcpvmlnxvsc`.\n   1. Set the variable `gcp_project` to your GCP `project` ID.\n   1. Set the variable `gcp_vmname` to your GCP `region` ID, e.g. us-east1.\n   1. Set the variable `gcp_zone` to your GCP `zone` ID, e.g. us-east1-d.\n   1. Set the variable `vm_pwd` to the password that you will use to ssh to the VM.\n   1. Set the variable `gcp_vm_username` to the first portion (before the \"@\" sign) of the email associated to your GCP account, which should be your GitLab email.\n\n### Run the project pipeline\n\n\nAfter configuring the second GitLab project, manually trigger the\nprovisioning of VS Code and its dependencies to the GCP VM as follows:\n\n\n1. On the left sidebar, select **Build > Pipelines** and click on the button\n**Run Pipeline**. On the next screen, click on the button **Run pipeline**.\n\n    The pipeline will:\n\n    - install `xauth` on the virtual machine. This is needed for effective X11 communication between your local desktop and the VM \n    - install `git` on the VM\n    - install `Visual Studio Code` on the VM.\n\n2. At this point, you can wait until the pipeline successfully completes. If\nyou don't want to wait, you can continue to do the first step of the next\nsection. However, you must ensure the pipeline has successfully completed\nbefore you can perform Step 2 of the next section.\n\n\n### Connect to your VM from your local Mac laptop\n\n\nNow that you have an instance of VS Code running on a Linux VM on GCP, you\nneed to configure your Mac laptop to be able to act as a client to the\nremote VM. Follow these steps:\n\n\n1. To connect to the remote VS Code from your Mac, you must first install\n`XQuartz` on your Mac. You can execute the following command on your Mac to\ninstall it:\n\n\n```\n\nbrew install xquartz\n\n```\n\nOr, you can follow the instructions from the following\n[tutorial](https://und.edu/research/computational-research-center/tutorials/mac-x11.html)\nfrom the University of North Dakota.\n\n\nAfter the pipeline for project `vscvmsetup` successfully executes to\ncompletion (pipeline you manually executed in the previous section), you can\nconnect to the remote VS Code as follows:\n\n\n2. Launch `XQuartz` on your Mac (it should be located in your Applications\nfolder). Its launching should open up an `xterm` on your Mac. If it does\nnot, then you can select **Applications > Terminal** from the `XQuartz` top\nmenu. \n\n3. On the `xterm`, enter the following command:\n\n\n```\n\ngcloud compute ssh --zone \"[GCP zone]\" \"[name of your VM]\" --project \"[GCP\nproject]\" --ssh-flag=\"-Y\"\n\n```\n\nWhere:\n\n\n- `[VM name]` is the name of the VM you created in project `gcpvmlnxvsc`.\nIts value should be the same as the `gcp_project` variable.\n\n- `[GCP zone]` is the zone where the VM is running. Its value should be the\nsame as the `gcp_vmname` variable.\n\n- `[GCP project]` is the name of your GCP project assigned name. Its value\nshould be the same as the `gcp_project` variable.\n\n\n***Note: If you have not installed the Google Cloud CLI, please do so by\nfollowing the [Google\ndocumentation](https://cloud.google.com/sdk/docs/install).***\n\n\n4. If you have not used SSH on your Mac before, you may not have a `.ssh` in\nyour `HOME` directory. If this is the case, you will be asked if you would\nlike to continue with the creation of this directory. Answer **Y**.\n\n\n5. Next, you will be asked to enter the same password twice to generate a\npublic/private key. Enter the same password you used when defining the\nvariable `vm_pwd` in the required configuration above.\n\n\n6. Once the SSH key is done propagating, you will need to enter the password\nagain two times to log in to the VM.\n\n\n7. You should now be logged in to the VM.\n\n\n### Create a personal access token\n\n\nThe assumption here is that you already have a GitLab project that you would\nwant to open from and work on the remote VS Code. To do this, you will need\nto clone your GitLab project from the VM. First, you will be using a\npersonal access token (PAT) to clone your project.\n\n\n1. Head over to your GitLab project (the one that you'd like to open from\nthe remote VS Code).\n\n2. From your GitLab project, create a\n[PAT](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token),\nname it `pat-gcpvm` and ensure that it has the following scopes:\n`read_repository`, `write_repository`, `read_registry`, `write_registry`,\nand `ai_features`\n\n3. Save the generated PAT somewhere safe; you will need it later.\n\n\n### Clone the read_repository\n\n\n1. On your local Mac, from the `xterm` where you are logged on to the remote\nVM, enter the following command:\n\n\n```\n\ngit clone https://[your GitLab\nusername]:[personal_access_token]@gitlab.com/[GitLab project name].git \n\n```\n\n\nWhere:\n\n\n- `[your GitLab username]` is your GitLab handle.\n\n- `[personal_access_token]` is the PAT you created in the previous section.\n\n- `[GitLab project name]` is the name of the project that contains the\nGitLab Code Suggestions test cases.\n\n\n## Launch Visual Studio Code\n\n\n1. From the `xterm` where you are logged in to the VM, enter the following\ncommand:\n\n\n```\n\ncode\n\n```\n\n\nWait for a few seconds and Visual Studio Code will appear on your Mac\nscreen.\n\n\n2. From the VS Code menu, select **File > Open Folder...\"\n\n3. In the File chooser, select the top-level directory of the GitLab project\nyou cloned in the previous section\n\n\nThat's it! You're ready to start working on your cloned GitLab project using\nthe VS Code that you installed on a remote Linux-based VM.\n\n\n### Troubleshooting\n\n\nWhile using the remotely installed VS Code from your local Mac, you may\nencounter a few issues. In this section, we provide guidance on how to\nmitigate them.\n\n\n#### Keyboard keys not mapped correctly\n\n\nIf, while running VS Code, you are having issues with your keyboard keys not\nbeing mapped correctly, e.g. letter e is backspace, letter r is tab, letter\ns is clear line, etc., do the following:\n\n\n1. In VS Code, select **File > Preferences > Settings**.\n\n1. Search for \"keyboard\". If having issues with the letter e, then search\nfor \"board\". Click on the \"Keyboard\" entry under \"Application.\"\n\n1. Ensure that the Keyboard Dispatch is set to \"keyCode.\"\n\n1. Restart VS Code.\n\n1. If you need further help, this is a good resource for [keyboard\nproblems](https://github.com/microsoft/vscode/wiki/Keybinding-Issues#troubleshoot-linux-keybindings).\n\n\n#### Error loading webview: Error\n\n\nIf while running VS Code, you get a message saying:\n\n\n\"Error loading webview: Error: Could not register service worker:\nInvalidStateError: Failed to register a ServiceWorker: The document is in an\ninvalid state.\"\n\n\n1. Exit VS Code and then enter this cmd from the `xterm` window:\n\n\n`killall code`\n\n\nYou may need to execute this command two or three times in a row to kill all\nVS Code processes.\n\n\n2. Ensure that all VS Code-related processes are gone by entering the\nfollowing command from the `xterm` window:\n\n\n`ps -ef | grep code`\n\n\n3. Once all the VS Code-related processes are gone, restart VS Code by\nentering the following command from the `xterm` window:\n\n\n`code`\n\n\n#### Some useful commands to debug SSH\n\n\nHere are some useful commands to run on the VM that can help you debug SSH\nissues:\n\n\n1. To get the status, location and latest event of sshd:\n\n\n`sudo systemctl status ssh`\n\n\n2. To see the log of sshd:\n\n\n`journalctl -b -a -u ssh`\n\n\n3. To restart to SSH daemon:\n\n\n`sudo systemctl restart ssh.service`\n\n\nOr\n\n\n`sudo systemctl restart ssh`\n\n\n4. To start a root shell:\n\n\n`sudo -s`\n\n\n## Get started\n\n\nThis article described how to:\n\n- instantiate a Linux-based VM on GCP\n\n- install VS Code and dependencies on the remote VM\n\n- clone an existing GitLab project of yours in the remote VM\n\n- open your remotely cloned project from the remotely installed VS Code\n\n\nAs a result, you can basically use your laptop as a thin client that\naccesses a remote server, where all the work takes place.\n\n\n> The automation to get all these parts in place was done by GitLab. Sign up\nfor a [free GitLab Ultimate\ntrial](https://about.gitlab.com/free-trial/) to get started today!\n",[1898,9,743],{"slug":4381,"featured":90,"template":700},"tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access","content:en-us:blog:tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access.yml","Tutorial Install Vs Code On A Cloud Provider Vm And Set Up Remote Access","en-us/blog/tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access.yml","en-us/blog/tutorial-install-vs-code-on-a-cloud-provider-vm-and-set-up-remote-access",{"_path":4387,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4388,"content":4394,"config":4400,"_id":4402,"_type":13,"title":4403,"_source":15,"_file":4404,"_stem":4405,"_extension":18},"/en-us/blog/tutorial-integrate-gitlab-merge-request-approvals-with-external-systems",{"title":4389,"description":4390,"ogTitle":4389,"ogDescription":4390,"noIndex":6,"ogImage":4391,"ogUrl":4392,"ogSiteName":685,"ogType":686,"canonicalUrls":4392,"schema":4393},"Tutorial: Integrate GitLab Merge Request approvals with external systems","Learn how to improve GitLab extensibility and integration with external applications in this demo. The result: a seamless integration that provides more control over merge requests.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749676011/Blog/Hero%20Images/blog-image-template-1800x945.svg","https://about.gitlab.com/blog/tutorial-integrate-gitlab-merge-request-approvals-with-external-systems","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Integrate GitLab Merge Request approvals with external systems\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Samer Akkoub\"}],\n        \"datePublished\": \"2024-10-08\",\n      }",{"title":4389,"description":4390,"authors":4395,"heroImage":4391,"date":4397,"body":4398,"category":694,"tags":4399},[4396],"Samer Akkoub","2024-10-08","GitLab customers often ask how to connect merge requests to external applications, such as ServiceNow or custom-built applications, to control approvals for the merging of code into a target branch from these external systems. To address this need, GitLab offers [External Status Check](https://docs.gitlab.com/ee/user/project/merge_requests/status_checks.html), a powerful feature that allows the sending of API calls to external systems to request the status of an external requirement, providing seamless integration and control over your merge requests.\n\nIn this article, I'll demonstrate this feature by explaining how to deploy an application I developed. The application is designed to receive status check requests from GitLab Merge Requests, list them, and enable external users to approve/reject these requests without logging in to the GitLab console. As a result, GitLab platform architects will better understand GitLab extensibility and integration with external systems.\n\nThe provided sample application can:\n1. Receive API requests from merge requests.\n2. Store the requests in AlchemyDB running on the same instance.\n3. Show Approve/Reject buttons for each row to approve or reject the corresponding merge request status check.\n\n## How to deploy the status review demo application\n1. Import this [GitLab repo project](https://gitlab.com/sakkoub-publicgroup/external-approval-app) to your GitLab account.\n2. The project pipeline will deploy the application to a Kubernetes cluster. To achieve this, define a [GitLab Agent](https://docs.gitlab.com/ee/user/clusters/agent/install/index.html) for Kubernetes in a separate project and include a path to the cloned project under the “[user_access](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html)” section in the agent configuration.\n3. Add a new environment variable `KUBE_CONTEXT`, with the value equal to the used agent path:name, similar to the following structure `path/to/agent/project:agent-name`.\n4. The status check application will be deployed to the `approval-app` namespace by default.\n5. Create the `approval-app` namespace in the target Kubernetes cluster.\n6. In the created namespace, add a secret named `gitlab-token` with the value set to the personal access token (PAT) of the user who will be approving the requests. The approval application will use this PAT to communicate back to the GitLab instance.\n7. Run the status check application pipeline on the main branch.\n8. Once deployed, the application will be exposed behind a load balancer. Use this command to grab the public IP address of the load balancer: `kubectl get services -n approval-app`.\n9. The application can then be accessed using this URL: http://EXTERNAL-IP/approval-apps/. Replace the `EXTERNAL-IP` with the value of the external IP address from the previous step. The resulting page should look like below (the table would be empty as we have not added any new merge requests yet).\n\n![Table showing IP address](https://res.cloudinary.com/about-gitlab-com/image/upload/v1752507534/v0pgvobf09eh9yqxqzrk.png)\n\n## Configure status check in GitLab\n\n1. In the GitLab project where the external status check needs to be configured, from the left menu, navigate under settings **-\\> Merge Request** and scroll down to **Status checks**.\n2. Click on **Add status check**.\n3. Add a service name.\n4. For the API to check enter: `[http://EXTERNAL-IP[/approval-apps/status_check`. Replace the `EXTERNAL-IP` with the external IP address found in the previous steps.\n5. Leave the `Target Branch` to the default, or select branch if you want this check to be triggered only for merge requests against certain branches.\n6. Leave `HMAC Shared Secret` as it is and click **Add status check**.\n\n![How to configure status check](https://res.cloudinary.com/about-gitlab-com/image/upload/v1752507426/jal2hw9ef3pydbetbp7p.png)\n\n## Test everything together\n\n1. In the project where you have configured the external check, create a new merge request from any branch targeting the main branch (assuming the main branch was selected when the external check was configured in the previous section).\n2. In the merge request details, look for the **Status checks** section and it should show `1 Pending`.\n3. Now, in a new tab, open the deployed external check application using this URL (replace `EXTERNAL-IP` with the value of the external IP address from the previous steps): `http://EXTERNAL-IP/approval-apps/`.\n4. A new entry should show in the list for the request external check from the merge request just created. Click on **Approve**.\n5. Switch back to the merge request's details screen and notice how the merge request is showing an approved status now.\n\n## Debugging tips\n\nUse the following notes to debug if something does not go as planned:\n\nIt is always helpful to view the logs for the external status check application. To do so: \n   1. Extract the name of the application pod using this command: `kubectl get pods -n approval-app`.\n   2. View the pod logs `kubectl logs [THE NAME OF THE POD] -n approval-app`.\n\nYou can SSH into the application pod and view the database (Alchemydb), which is used for the application. \n   1. `kubectl exec -it \\[POD-NAME\\] -n approval-app -- /bin/sh` \n   2. `cd instance`\n   3. `sqlite3 gitlab_status_checks.db` \n   4. To view the database tables, type `.tables`.\n   5. To describe the table structure, type `PRAGMA table_info('status_check');`.\n   6. To view all the records in the `status_check` table, type `select * from status_check`.\n\n> Discover more about [GitLab External Status Check](https://docs.gitlab.com/ee/user/project/merge_requests/status_checks.html) and how to gain more control over merge requests.\n",[9,694,696,108],{"slug":4401,"featured":6,"template":700},"tutorial-integrate-gitlab-merge-request-approvals-with-external-systems","content:en-us:blog:tutorial-integrate-gitlab-merge-request-approvals-with-external-systems.yml","Tutorial Integrate Gitlab Merge Request Approvals With External Systems","en-us/blog/tutorial-integrate-gitlab-merge-request-approvals-with-external-systems.yml","en-us/blog/tutorial-integrate-gitlab-merge-request-approvals-with-external-systems",{"_path":4407,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4408,"content":4414,"config":4419,"_id":4421,"_type":13,"title":4422,"_source":15,"_file":4423,"_stem":4424,"_extension":18},"/en-us/blog/tutorial-migrate-from-google-cloud-source-repositories-to-gitlab",{"title":4409,"description":4410,"ogTitle":4409,"ogDescription":4410,"noIndex":6,"ogImage":4411,"ogUrl":4412,"ogSiteName":685,"ogType":686,"canonicalUrls":4412,"schema":4413},"Tutorial: Migrate from Google Cloud Source Repositories to GitLab","Google Cloud is deprecating Cloud Source Repositories. Learn how to migrate a CSR source code repository to GitLab, along with best practices.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097739/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2813%29_1zdtbfPDHZVe6JC2AbdHmb_1750097738370.png","https://about.gitlab.com/blog/tutorial-migrate-from-google-cloud-source-repositories-to-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Migrate from Google Cloud Source Repositories to GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tsukasa Komatsubara\"},{\"@type\":\"Person\",\"name\":\"Regnard Raquedan\"}],\n        \"datePublished\": \"2024-08-28\",\n      }",{"title":4409,"description":4410,"authors":4415,"heroImage":4411,"date":716,"body":4417,"category":694,"tags":4418},[4416,1874],"Tsukasa Komatsubara","Google Cloud’s [deprecation of Cloud Source\nRepositories](https://cloud.google.com/source-repositories/docs/release-notes)\n(CSR) has prompted development teams to seek a full-featured alternative for\ntheir source code repositories. GitLab, a [Google Cloud Technology\nPartner](https://cloud.google.com/find-a-partner/partner/gitlab-inc), is a\nstrong choice due to its comprehensive DevSecOps capabilities.\n\n\nIn this tutorial, you'll learn the steps to ensure a smooth transition from\nCSR to GitLab, whether you're using GitLab.com or a self-managed instance on\nGoogle Cloud.\n\n\n## Why GitLab?\n\nTransitioning from Google Cloud Source Repositories to GitLab is a\nrecommended step. As a strategic partner of Google Cloud, GitLab seamlessly\nintegrates with existing infrastructure with ease and brings value to\ncustomers in the following ways:\n\n- **Unified DevSecOps platform**\n    - Consolidate your entire development lifecycle into a single application, from planning to monitoring. Eliminate tool sprawl and dramatically boost productivity.\n- **Seamless Google Cloud integration**\n    - Effortlessly connect with GKE, Cloud Build, and Cloud Storage, ensuring a smooth migration and efficient operations within the Google Cloud ecosystem.\n- **Advanced CI/CD capabilities**\n    - Leverage [Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/) to automate everything from security scanning to deployment, accelerating your development cycles.\n- **Industry-recognized AI coding assistance**\n    - Benefit from built-in AI-assisted development with [GitLab Duo](https://about.gitlab.com/gitlab-duo/), fostering a secure and efficient coding environment.\n\n## Prerequisites\n\n\nBefore you start the migration, ensure you have:\n\n- GitLab account: Set up your account on GitLab.com or on a self-hosted\ninstance.\n\n- GitLab project: Create a blank project in GitLab where the CSR repository\nwill be migrated.\n\n\n## Migration steps\n\n\n1. Create a blank GitLab project: This will serve as the destination for\nyour migrated CSR repository. Keep this project empty for now.\n\n2. Generate a personal access token (PAT): Navigate to GitLab settings and\n[generate a\nPAT](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)\nwith `read_repository` and `write_repository` scopes enabled. This token\nwill be used to authenticate your Git operations during the migration\nprocess.\n\n3. Edit code in Cloud Shell Editor: From your CSR repository, open the Cloud\nShell Editor by clicking the “Edit code” button. You’ll need to authorize\nthe Cloud Shell and select “Trust repo” to proceed.\n\n\n![Google Cloud Shell\nEditor](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097750/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097750517.png)\n\n\n4. Inspect Git status: Run `git status` in the Cloud Shell to check the\ncurrent branch and ensure everything is in order before pushing to GitLab.\n\n\n![Inspect Git\nstatus](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097750/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097750518.png)\n\n\n5. Set Up the Remote Repository: Add your GitLab project as a remote\nrepository by running:\n\n\n```\n\ngit remote add origin [GITLAB_PROJECT_URL]\n\n\n```\n\n\n6. Replace `[GITLAB_PROJECT_URL]` with the actual URL of your GitLab\nproject.\n\nPush to GitLab: Finally, push your local repository to GitLab by running: \n\n\n```\n\ngit push -u origin [BRANCH_NAME]\n\n\n```\n\n\n7. Replace `[BRANCH_NAME]` with the current branch name you noted earlier.\n\nWhen prompted, use your GitLab username and the PAT as the password to\nauthenticate and complete the push.\n\n\n## Best practices\n\n\n- Back up before you begin: Always back up your CSR repository before\nstarting the migration process.\n\n- Test after migration: Ensure all aspects of the repository, including\nbranches and CI/CD pipelines, are functioning as expected in GitLab.\n\n- Leverage GitLab features: Take advantage of GitLab’s advanced DevSecOps\nfeatures such as [AI](https://about.gitlab.com/gitlab-duo/),\n[CI/CD](https://docs.gitlab.com/ee/ci/), and [Enterprise Agile\nplanning](https://about.gitlab.com/solutions/agile-delivery/) to enhance\nyour development workflow.\n\n\nMoving from Google Cloud Source Repositories to GitLab is easy and offers\nmore benefits than just managing source code. GitLab, with its integration\nwith Google Cloud, makes it an ideal choice for developers seeking to\nenhance their workflow post-migration.\n\n\n> Read more about [GitLab's integration with Google\nCloud](https://about.gitlab.com/blog/gitlab-google-cloud-integrations-now-in-public-beta/).\n",[9,1264,495],{"slug":4420,"featured":6,"template":700},"tutorial-migrate-from-google-cloud-source-repositories-to-gitlab","content:en-us:blog:tutorial-migrate-from-google-cloud-source-repositories-to-gitlab.yml","Tutorial Migrate From Google Cloud Source Repositories To Gitlab","en-us/blog/tutorial-migrate-from-google-cloud-source-repositories-to-gitlab.yml","en-us/blog/tutorial-migrate-from-google-cloud-source-repositories-to-gitlab",{"_path":4426,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4427,"content":4432,"config":4437,"_id":4439,"_type":13,"title":4440,"_source":15,"_file":4441,"_stem":4442,"_extension":18},"/en-us/blog/tutorial-secure-bigquery-data-publishing-with-gitlab",{"title":4428,"description":4429,"ogTitle":4428,"ogDescription":4429,"noIndex":6,"ogImage":683,"ogUrl":4430,"ogSiteName":685,"ogType":686,"canonicalUrls":4430,"schema":4431},"Tutorial: Secure BigQuery data publishing with GitLab ","Learn how to create repeatable, auditable, and efficient processes for automating and securing BigQuery data exports.","https://about.gitlab.com/blog/tutorial-secure-bigquery-data-publishing-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Secure BigQuery data publishing with GitLab \",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Regnard Raquedan\"}],\n        \"datePublished\": \"2025-03-25\",\n      }",{"title":4428,"description":4429,"authors":4433,"heroImage":683,"date":4434,"body":4435,"category":785,"tags":4436},[1874],"2025-03-25","GitLab offers a powerful solution for automating and securing\n[BigQuery](https://cloud.google.com/bigquery) data exports. This integration\ntransforms manual exports into repeatable, auditable processes that can\neliminate security vulnerabilities while saving valuable time. This tutorial\nexplains how to implement this solution so you can quickly reduce manual\noperations, permission issues, and security concerns with just a few lines\nof GitLab YAML code.\n\n\nFollow along with this step-by-step video:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/gxXX-ItAreo?si=FijY9wMVppCW-18q\" frameborder=\"0\" allowfullscreen=\"true\">\u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n## The solution architecture\n\n\nOur solution leverages GitLab CI/CD pipelines to automate the secure export\nof data from BigQuery to Google Cloud Storage. Here's the high-level\narchitecture:\n\n\n1. SQL code is stored and version-controlled in GitLab.  \n\n2. After code review and approval, GitLab CI/CD pipeline executes the\ncode.  \n\n3. The pipeline authenticates with Google Cloud.  \n\n4. SQL queries are executed against BigQuery.  \n\n5. Results are exported as CSV files to Google Cloud Storage.  \n\n6. Secure links to these files are provided for authorized consumption.\n\n\n## Prerequisites\n\n\nBefore we begin, ensure you have:\n\n\n* **Google Cloud APIs enabled:** BigQuery API and Cloud Storage API  \n\n* **Service account** with appropriate permissions:  \n  * BigQuery Job User  \n  * Storage Admin  \n  * **Note:** For this demo, we're using the service account approach for authentication, which is simpler to set up. For production environments, you might consider using GitLab's identity and access management integration with Google Cloud. This integration leverages Workload Identity Federation, which provides enhanced security and is more suitable for enterprise customers and organizations.  \n* **GitLab project** ready to store your SQL code and pipeline configuration\n\n\n## Step-by-step implementation\n\n\n**1. Configure Google Cloud credentials.**\n\n\nFirst, set up the necessary environment variables in your GitLab project:\n\n\n- Go to your **GitLab project > Settings > CI/CD**.  \n\n- Expand the **Variables** section.  \n\n- Add the following variables:  \n   * `GCS_BUCKET`: Your Google Cloud Storage bucket name  \n   * `GCP_PROJECT_ID`: Your Google Cloud project ID  \n   * `GCP_SA_KEY`: Base64-encoded service account key (mark as masked)\n\n**2. Create your SQL query.**\n\n\nCreate a file named `query.sql` in your GitLab repository with your BigQuery\nSQL query. The query looks like this:\n\n\n```\n\n-- This query shows a list of the daily top Google Search terms.\n\nSELECT\n   refresh_date AS Day,\n   term AS Top_Term,\n       -- These search terms are in the top 25 in the US each day.\n   rank,\nFROM `bigquery-public-data.google_trends.top_terms`\n\nWHERE\n   rank = 1\n       -- Choose only the top term each day.\n   AND refresh_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 2 WEEK)\n       -- Filter to the last 2 weeks.\nGROUP BY Day, Top_Term, rank\n\nORDER BY Day DESC\n   -- Show the days in reverse chronological order.\n\n```\n\n\nThis query gets the top 25 search terms from Google Trends for the current\nday.\n\n\n**3. Configure the GitLab CI/CD pipeline.**\n\n\nCreate a `.gitlab-ci.yml` file in your repository root:\n\n\n```\n\nimage: google/cloud-sdk:alpine\n\n\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml\n\nexecute:\n  stage: deploy\n  script: \n    # Set up Google Cloud authentication and install necessary components\n    - export GOOGLE_CLOUD_CREDENTIALS=$(echo $SERVICE_ACCOUNT_KEY | base64 -d)\n    - echo $GOOGLE_CLOUD_CREDENTIALS > service-account-key.json \n    - gcloud auth activate-service-account --key-file service-account-key.json \n    - gcloud components install gsutil\n    # Set the active Google Cloud project\n    - gcloud config set project $PROJECT_ID\n    # Run the BigQuery query and export the results to a CSV file\n    - bq query --format=csv --use_legacy_sql=false \u003C test.sql > results.csv\n    # Create a Google Cloud Storage bucket if it doesn't exist\n    - gsutil ls gs://${CLOUD_STORAGE_BUCKET} || gsutil mb gs://${CLOUD_STORAGE_BUCKET}\n    # Upload the CSV file to the storage bucket\n    - gsutil cp results.csv gs://${CLOUD_STORAGE_BUCKET}/results.csv\n    # Set the access control list (ACL) to make the CSV file publicly readable\n    - gsutil acl ch -u AllUsers:R gs://${CLOUD_STORAGE_BUCKET}/results.csv\n    # Define the static URL for the CSV file\n    - export STATIC_URL=\"https://storage.googleapis.com/${CLOUD_STORAGE_BUCKET}/results.csv\"\n    # Display the static URL for the CSV file\n    - echo \"File URL = $STATIC_URL\"\n\n```\n\n\n**4. Run the pipeline.**\n\n\nNow, whenever changes are merged to your main branch, the pipeline will\nprovide a link to the CSV file stored on the Google Cloud Storage bucket.\nThis file contains the result of the executed SQL query that GitLab subjects\nto security checks.\n\n\n## Benefits of this approach\n\n\n* **Security:** Authentication is handled automatically via service accounts\n(or Workload Identity Federation for enhanced security in production\nenvironments).  \n\n* **Auditability:** All data exports are tracked through GitLab commits and\npipeline logs.  \n\n* **Repeatability:** Consistent, predictable export process on every run,\nand can be scheduled.  \n\n* **Version control:** SQL queries are properly versioned and reviewed.  \n\n* **Automation:** Significantly fewer manual exports, reducing human error.\n\n\n## Try it today\n\n\nBy combining GitLab's DevSecOps capabilities with Google Cloud's BigQuery\nand Cloud Storage, you've now automated and secured your data publishing\nworkflow. This approach reduces manual operations, resolves permission\nheadaches, and addresses security concerns – all achieved with just a few\nlines of GitLab CI code.\n\n\n> Use this tutorial's [complete code\nexample](https://gitlab.com/gitlab-partners-public/google-cloud/demos/big-query-data-publishing)\nto get started now.\n",[851,495,9,956,232,1264],{"slug":4438,"featured":90,"template":700},"tutorial-secure-bigquery-data-publishing-with-gitlab","content:en-us:blog:tutorial-secure-bigquery-data-publishing-with-gitlab.yml","Tutorial Secure Bigquery Data Publishing With Gitlab","en-us/blog/tutorial-secure-bigquery-data-publishing-with-gitlab.yml","en-us/blog/tutorial-secure-bigquery-data-publishing-with-gitlab",{"_path":4444,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4445,"content":4451,"config":4456,"_id":4458,"_type":13,"title":4459,"_source":15,"_file":4460,"_stem":4461,"_extension":18},"/en-us/blog/tutorial-security-scanning-in-air-gapped-environments",{"title":4446,"description":4447,"ogTitle":4446,"ogDescription":4447,"noIndex":6,"ogImage":4448,"ogUrl":4449,"ogSiteName":685,"ogType":686,"canonicalUrls":4449,"schema":4450},"Tutorial: Security scanning in air-gapped environments","Security scanning remains crucial even in air-gapped environments to detect internal threats, prevent data exfiltration, and maintain operational integrity. Learn how GitLab can help get air-gapped environments secure.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099301/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_1097303277_6gTk7M1DNx0tFuovupVFB1_1750099300786.jpg","https://about.gitlab.com/blog/tutorial-security-scanning-in-air-gapped-environments","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Tutorial: Security scanning in air-gapped environments\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-02-05\",\n      }",{"title":4446,"description":4447,"authors":4452,"heroImage":4448,"date":4453,"body":4454,"category":830,"tags":4455},[1444],"2025-02-05","Air-gapped environments are computer networks or systems that are physically\nisolated from unsecured networks, such as the public internet or unsecured\nlocal area networks. This isolation is implemented as a security measure to\nprotect sensitive data and critical systems from external cyber threats by\nproviding:\n\n\n* Enhanced security: By physically isolating systems from external networks,\nair-gapped environments help prevent remote attacks, malware infections, and\nunauthorized data access. This is crucial for highly sensitive data and\ncritical systems.\n\n* Data protection: Air-gapping provides the strongest protection against\ndata exfiltration since there's no direct connection that attackers could\nuse to steal information.\n\n* Critical infrastructure protection: For systems that control vital\ninfrastructure (like power plants, water treatment facilities, or military\nsystems), air-gapping helps prevent potentially catastrophic cyber attacks.\n\n* Compliance requirements: Many regulatory frameworks require air-gapping\nfor certain types of sensitive data or critical systems, particularly in\ngovernment, healthcare, and financial sectors.\n\n* Malware protection: Without network connectivity, systems are protected\nfrom network-based malware infections and ransomware attacks.\n\n\nEven though air-gapped systems are isolated, they can still have\nvulnerabilities. Regular security scanning helps identify these weaknesses\nbefore they can be exploited. In this article, you will learn the different\nsecurity scanners GitLab provides and how they can be added/updated in a\nlimited-connectivity environment.\n\n\n## GitLab security scanners in air-gapped environments\n\n\nGitLab provides a variety of different security scanners for the complete\napplication lifecycle. The scanners that support air-gapped environments\ninclude:\n\n\n* [Static Application Security Testing\n(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/index.html#running-sast-in-an-offline-environment)  \n\n* [Dynamic Application Security Testing\n(DAST](https://docs.gitlab.com/ee/user/application_security/dast/browser/configuration/offline_configuration.html))  \n\n* [Secret\nDetection](https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/index.html#offline-configuration)  \n\n* [Container\nScanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#running-container-scanning-in-an-offline-environment)  \n\n* [Dependency\nScanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#offline-environment)  \n\n* [API\nFuzzing](https://docs.gitlab.com/ee/user/application_security/api_fuzzing/configuration/offline_configuration.html)  \n\n* [License\nScanning](https://docs.gitlab.com/ee/user/compliance/license_scanning_of_cyclonedx_files/index.html#running-in-an-offline-environment)\n\n\nBy default, GitLab Self-Managed instances pull security scanner images from\nthe public GitLab container registry (registry.gitlab.com) and store them\nwithin the [built-in local GitLab container\nregistry](https://docs.gitlab.com/ee/user/packages/container_registry/). I\nwill demonstrate this flow below by running the following pipeline that\nscans for secrets on a [sample\nproject](https://gitlab.com/gitlab-da/tutorials/security-and-governance/owasp/juice-shop): \n\n\n```yaml\n\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n```\n\n\nWhen running the job in an internet-connected GitLab instance the job\npasses:\n\n\n![GitLab Runner with internet access successfully pulling from external\nregistry\n\n](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099328/Blog/Content%20Images/Blog/Content%20Images/pass-1_aHR0cHM6_1750099328577.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Runner with internet access successfully pulling from\nexternal registry\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\nHowever, If I disable internet access to the VM running GitLab, the\n`secret-detection` job will fail to download the container image, causing\nthe job to fail:\n\n\n![GitLab Runner without internet access failing to pull from external\nregistry](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099328/Blog/Content%20Images/Blog/Content%20Images/fail-1_aHR0cHM6_1750099328577.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Runner without internet access failing to pull from\nexternal registry\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nAlternatively, if I set my GitLab Runners’ pull image policy to\n`if-not-present` from `always`, I can load the cached version of the scanner\nif it was run before on the internet by using the image stored in our local\ndocker:\n\n\n![GitLab Runner without internet access successfully pulling from internal\nregistry\ncache](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099329/Blog/Content%20Images/Blog/Content%20Images/pass-2_aHR0cHM6_1750099328579.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Runner without internet access successfully pulling from\ninternal registry cache\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\n### Setting up offline scanning prerequisites\n\n\nRunning these security scanners in an air-gapped environment requires the\nfollowing:\n\n\n* [GitLab Ultimate\nsubscription](https://about.gitlab.com/pricing/ultimate/)  \n\n* [Offline cloud\nlicense](https://about.gitlab.com/pricing/licensing-faq/cloud-licensing/#offline-cloud-licensing)  \n\n* GitLab Self-Managed cluster\n\n\nYou can follow along with this tutorial in any GitLab Self-Managed EE\ninstance (even those that are not air-gapped) to learn how to transfer and\nrun images in an air-gapped environment. In this tutorial, I will\ndemonstrate how to load scanner images onto a GitLab-EE instance running in\na Google Compute VM where I cut off the `EGRESS` to everything by\nimplementing firewall rules:\n\n\n```bash\n\n# egress firewall rule to block all outbound traffic to the internet\n\n$ gcloud compute firewall-rules create deny-internet-egress \\\n    --direction=EGRESS \\\n    --priority=1000 \\\n    --network=default \\\n    --action=DENY \\\n    --rules=all \\\n    --destination-ranges=0.0.0.0/0 \\\n    --target-tags=no-internet\n\n# Create an allow rule for internal traffic with higher priority\n\n$ gcloud compute firewall-rules create allow-internal-egress \\\n    --direction=EGRESS \\\n    --priority=900 \\\n    --network=default \\\n    --action=ALLOW \\\n    --rules=all \\\n    --destination-ranges=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 \\\n    --target-tags=no-internet\n\n# Apply tag to VM\n\n$ gcloud compute instances add-tags YOUR_VM_NAME \\\n    --zone=YOUR_ZONE \\\n    --tags=no-internet\n```\n\n\nThen, once I SSH into my VM, you can see we cannot connect to\nregistry.gitlab.com:\n\n\n```bash\n\n# showing I can’t access the gitlab container registry\n\n$ ping registry.gitlab.com\n\nPING registry.gitlab.com (35.227.35.254) 56(84) bytes of data.\n\n^C\n\n--- registry.gitlab.com ping statistics ---\n\n3 packets transmitted, 0 received, 100% packet loss, time 2031ms\n\n```\n\n\n**Note:** I am still allowing ingress so I can copy files and SSH into the\nmachine.\n\n\n## Load security scanners in air-gapped environments\n\n\nTo use the various security scanners on air-gapped environments, the GitLab\nRunner must be able to fetch the scanner container images from GitLab’s\nbuilt-in container registry. This means that the container images for the\nsecurity scanners must be downloaded and packaged in a separate environment\nwith access to the public internet. The process of loading security scanners\nonto an air-gapped environment includes the following:\n\n\n1. Download and package container images from the public internet.\n\n2. Transfer images to offline environment.\n\n3. Load transferred images into offline container registry.\n\n\nNow let’s go over how we can implement GitLab Secret Detection in an\nair-gapped environment.\n\n\n### Download and package container images from public internet\n\n\nLet’s download the container image for secret detection and store it within\nour local container registry. Other scanner images can be found in the\n[offline deployments\ndocumentation](https://docs.gitlab.com/ee/user/application_security/offline_deployments/).\nI will be using Podman desktop to download these images, but you can use\nDocker desktop or other alternatives.\n\n\n1. Pull the GitLab Secret Detection image.\n\n\n```bash\n\n$ podman pull registry.gitlab.com/security-products/secrets:6\n\nTrying to pull registry.gitlab.com/security-products/secrets:6...\n\nGetting image source signatures\n\nCopying blob\nsha256:999745130ac045f2b1c29ecce088b43fc4a95bbb82b7960fb7b8abe0e3801bf8\n\nCopying blob\nsha256:a4f7c013bb259c146cd8455b7c3943df7ed84b157e42a2348eef16546d8179b1\n\nCopying blob\nsha256:1f3e46996e2966e4faa5846e56e76e3748b7315e2ded61476c24403d592134f0\n\nCopying blob\nsha256:400a41f248eb3c870bd2b07073632c49f1e164c8efad56ea3b24098a657ec625\n\nCopying blob\nsha256:9090f17a5a1bb80bcc6f393b0715210568dd0a7749286e3334a1a08fb32d34e6\n\nCopying blob\nsha256:c7569783959081164164780f6c1b0bbe1271ee8d291d3e07b2749ae741621ea3\n\nCopying blob\nsha256:20c7ca6108f808ad5905f6db4f7e3c02b21b69abdea8b45abfa34c0a2ba8bdb5\n\nCopying blob\nsha256:e8645a00be64d77c6ff301593ce34cd8c17ffb2b36252ca0f2588009a7918d2e\n\nCopying config\nsha256:0235ed43fc7fb2852c76e2d6196601968ae0375c72a517bef714cd712600f894\n\nWriting manifest to image destination\n\nWARNING: image platform (linux/amd64) does not match the expected platform\n(linux/arm64)\n\n0235ed43fc7fb2852c76e2d6196601968ae0375c72a517bef714cd712600f894\n\n\n$ podman images\n\nREPOSITORY                                                  TAG        \nIMAGE ID      CREATED      SIZE\n\nregistry.gitlab.com/security-products/secrets               6          \n0235ed43fc7f  4 hours ago  85.3 MB\n\n```\n\n\n2. Save the image as a tarball.\n\n\n```bash\n\n$ podman save -o secret-detection.tar\nregistry.gitlab.com/security-products/secrets:6\n\n$ chmod +r secret-detection.tar\n\n$ ls -al secret-detection.tar\n\n-rw-r--r--@ 1 fern  staff  85324800 Jan 10 10:25 secret-detection.tar\n\n```\n\n\nAlternatively, you can use the [official GitLab\ntemplate](https://docs.gitlab.com/ee/user/application_security/offline_deployments/#using-the-official-gitlab-template)\non an environment with internet access to download the container images\nneeded for the security scanners and save them as job artifacts or push them\nto the container registry of the project where the pipeline is executed. \n\n\n### Transfer images to offline environment\n\n\nNext, let's transfer the tarball to our air-gapped environment. This can be\ndone in several ways, depending on your needs, such as:\n\n\n* Physical media transfer  \n\n* Data diodes  \n\n* Guard systems  \n\n* Cross-domain solutions (CDS) \n\n\nI will SCP (Secure Copy Protocol) the tarball directly to my VM that does\nnot have egress access, but does allow ingress. As this is just for\ndemonstration purposes, make sure to consult your organization's security\npolicies and transfer procedures for air-gapped environments.\n\n\n#### Verify the image is not cached\n\n\nBefore transferring the file, I’ll delete the Docker images on my GitLab\ninstance pertaining to secret detection to make sure they aren't cached:\n\n\n```bash\n\n$ docker images\n\nREPOSITORY                                                         \nTAG              IMAGE ID       CREATED        SIZE\n\nregistry.gitlab.com/security-products/secrets                      \n6                0235ed43fc7f   9 hours ago    84.8MB\n\nregistry.gitlab.com/security-products/secrets                      \n\u003Cnone>           16d88433af61   17 hours ago   74.9MB\n\n\n$ docker image rmi 16d88433af61 -f\n\nUntagged:\nregistry.gitlab.com/security-products/secrets@sha256:f331da6631d791fcd58d3f23d868475a520f50b02d64000e2faf1def66c75d48\n\nDeleted:\nsha256:16d88433af618f0b405945031de39fe40b3e8ef1bddb91ca036de0f5b32399d7\n\nDeleted:\nsha256:1bb06f72f06810e95a70039e797481736e492201f51a03b02d27db055248ab6f\n\nDeleted:\nsha256:a5ef2325ce4be9b39993ce301f8ed7aad1c854d7ee66f26a56a96967c6606510\n\nDeleted:\nsha256:f7cdac818a36d6c023763b76a6589c0db7609ca883306af4f38b819e62f29471\n\nDeleted:\nsha256:5eabf4d47287dee9887b9692d55c8b5f848b50b3b7248f67913036014e74a0e9\n\nDeleted:\nsha256:51b7cb600604c0737356f17bc02c22bac3a63697f0bf95ba7bacb5b421fdb7da\n\nDeleted:\nsha256:1546193b011d192aa769a15d3fdd55eb4e187f201f5ff7506243abb02525dc06\n\nDeleted:\nsha256:1ea72408d0484c3059cc0008539e6f494dc829caa1a97d156795687d42d9cb57\n\nDeleted:\nsha256:1313ee9da7716d85f63cfdd1129f715e9bbb6c9c0306e4708ee73672b3e40f26\n\nDeleted:\nsha256:954ebfd83406f0dfed93eb5157ba841af5426aa95d4054174fff45095fd873a1\n\n\n$ docker image rmi 0235ed43fc7f -f\n\nUntagged: registry.gitlab.com/security-products/secrets:6\n\nDeleted:\nsha256:0235ed43fc7fb2852c76e2d6196601968ae0375c72a517bef714cd712600f894\n\nDeleted:\nsha256:f05f85850cf4fac79e279d93afb6645c026de0223d07b396fce86c2f76096c1f\n\nDeleted:\nsha256:7432b0766b885144990edd3166fbabed081be71d28d186f4d525e52729f06b1f\n\nDeleted:\nsha256:2c6e3361c2ee2f43bd75fb9c7c12d981ce06df2d51a134965fa47754760efff0\n\nDeleted:\nsha256:7ad7f7245b45fbe758ebd5788e0ba268a56829715527a9a4bc51708c21af1c7f\n\nDeleted:\nsha256:3b73a621115a59564979f41552181dce07f3baa17e27428f7fff2155042a1901\n\nDeleted:\nsha256:78648c2606a7c4c76885806ed976b13e4d008940bd3d7a18b52948a6be71b60d\n\nDeleted:\nsha256:383d4a6dc5be9914878700809b4a3925379c80ab792dfe9e79d14b0c1d6b5fad\n\n```\n\n\nThen I'll rerun the job to show the failure:\n\n\n![GitLab Runner without internet access fails to pull an image from internal\nregistry\ncache](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099328/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750099328580.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Runner without internet access fails to pull an image from\ninternal registry cache\u003C/i>\u003C/center>\n\n\n#### SCP file to GitLab instance\n\n\nNow, from my local machine, I will SCP the file to my GitLab instance as\nfollows:\n\n\n```bash\n\n$ gcloud compute scp secret-detection.tar INSTANCE:~ --zone=ZONE\n\nsecret-detection.tar                                                         \n100%   81MB  21.5MB/s   00:03\n\n```\n\n\n### Load transferred images into offline container registry\n\n\nNext, I'll SSH into my VM and load the Docker image:\n\n\n```bash\n\n$ gcloud compute ssh INSTANCE --zone=ZONE\n\n\n$ sudo docker load -i secret-detection.tar\n\nc3c8e454c212: Loading layer\n[==================================================>]  2.521MB/2.521MB\n\n51e93afaeedc: Loading layer\n[==================================================>]  32.55MB/32.55MB\n\ne8a25e39bb30: Loading layer\n[==================================================>]  221.2kB/221.2kB\n\n390704968493: Loading layer\n[==================================================>]  225.8kB/225.8kB\n\n76cf57e75f63: Loading layer\n[==================================================>]  17.64MB/17.64MB\n\nc4c7a681fd10: Loading layer\n[==================================================>]  4.608kB/4.608kB\n\nf0690f406157: Loading layer\n[==================================================>]  24.01MB/24.01MB\n\nLoaded image: registry.gitlab.com/security-products/secrets:6\n\n```\n\n\n### Run the scanners\n\n\nI'll [re-run the pipeline\nmanually](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually)\nand the scanner will be pulled from the cache. Once the pipeline completes,\nwe can see the secret detection job is successful:\n\n\n![GitLab Runner without internet access successfully pulling from internal\nregistry cache after image\nloaded](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099328/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750099328581.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Runner without internet access successfully pulling from\ninternal registry cache after image loaded\u003C/center>\u003C/i>\n\n\nIf you want to pull the image from a different location or you tag your\nimages in a different way, you can edit the config as follows:\n\n\n```yaml\n\ninclude:\n  - template: Jobs/Secret-Detection.gitlab-ci.yml\n\nvariables:\n  SECURE_ANALYZERS_PREFIX: \"localhost:5000/analyzers\"\n```\n\n\nSee the [offline environments\ndocumentation](https://docs.gitlab.com/ee/user/application_security/offline_deployments/)\nfor more information.\n\n\n### View scanner results\n\n\nOnce the scanner completes on the default branch, a vulnerability report is\npopulated with all the findings. The vulnerability report provides\ninformation about vulnerabilities from scans of the default branch.\n\n\nYou can access the vulnerability report by navigating to the side tab and\nselecting **Secure > Vulnerability Report**:\n\n\n![GitLab Vulnerability Report with secret detection\nfindings](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099328/Blog/Content%20Images/Blog/Content%20Images/vulnerability_report_aHR0cHM6_1750099328581.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Vulnerability Report with secret detection\nfindings\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\nThe project’s vulnerability report provides:\n\n- totals of vulnerabilities per severity level\n\n- filters for common vulnerability attributes\n\n- details of each vulnerability, presented in tabular layout\n\n- a timestamp showing when it was updated, including a link to the latest\npipeline\n\n\nWe can see that two vulnerabilities were detected by the Secret Detection\nscanner. If we click on a vulnerability, we will be transported to its\nvulnerability page:\n\n\n![GitLab Vulnerability Page showing detailed\ninsights](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750099329/Blog/Content%20Images/Blog/Content%20Images/insights_aHR0cHM6_1750099328582.png)\n\n\n\u003Ccenter>\u003Ci>GitLab Vulnerability Page showing detailed insights\u003C/center>\u003C/i>\n\n\n\u003Cbr>\u003C/br>\n\n\nThe vulnerability page provides details of the vulnerability, which can be\nused to triage and find a path to remediation. These vulnerability details\ninclude:\n\n- description\n\n- when it was detected\n\n- current status\n\n- available actions\n\n- linked issues\n\n- actions log\n\n- filename and line number of the vulnerability (if available)\n\n- severity\n\n\n## Read more\n\n\nTo learn more about GitLab and running security scanners in air-gapped\nenvironments, check out the following resources:\n\n\n* [GitLab Ultimate](https://about.gitlab.com/pricing/ultimate/)  \n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab Offline Deployments\nDocumentation](https://docs.gitlab.com/ee/user/application_security/offline_deployments/)  \n\n* [GitLab Application Security\nDocumentation](https://docs.gitlab.com/ee/user/application_security/)\n",[9,830,185,495,696],{"slug":4457,"featured":90,"template":700},"tutorial-security-scanning-in-air-gapped-environments","content:en-us:blog:tutorial-security-scanning-in-air-gapped-environments.yml","Tutorial Security Scanning In Air Gapped Environments","en-us/blog/tutorial-security-scanning-in-air-gapped-environments.yml","en-us/blog/tutorial-security-scanning-in-air-gapped-environments",{"_path":4463,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4464,"content":4469,"config":4474,"_id":4476,"_type":13,"title":4477,"_source":15,"_file":4478,"_stem":4479,"_extension":18},"/en-us/blog/ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation",{"title":4465,"description":4466,"ogTitle":4465,"ogDescription":4466,"noIndex":6,"ogImage":1153,"ogUrl":4467,"ogSiteName":685,"ogType":686,"canonicalUrls":4467,"schema":4468},"Ultimate guide to CI/CD: Fundamentals to advanced implementation","Learn how to modernize continuous integration/continuous deployment, including automating the development, delivery, and security of pipelines.","https://about.gitlab.com/blog/ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Ultimate guide to CI/CD: Fundamentals to advanced implementation\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Sandra Gittlen\"}],\n        \"datePublished\": \"2025-01-06\",\n      }",{"title":4465,"description":4466,"authors":4470,"heroImage":1153,"date":4471,"body":4472,"category":1119,"tags":4473},[4244],"2025-01-06","Continuous integration/continuous delivery ([CI/CD](https://about.gitlab.com/topics/ci-cd/)) has revolutionized how software teams create value for their users. Gone are the days of manual deployments and integration headaches — modern development demands automation, reliability, and speed.\n\nAt its core, CI/CD is about creating a seamless pipeline that takes code from a developer's environment all the way to production and incorporates feedback in real time. [CI](https://about.gitlab.com/topics/ci-cd/benefits-continuous-integration/) helps teams catch issues early — before they become costly problems — by ensuring that code changes are frequently merged into a shared repository, automatically tested, and validated. [CD](https://about.gitlab.com/topics/ci-cd/#what-is-continuous-delivery-cd) extends this by automating deployments, making releases predictable and stress-free.\n\nRather than relying on manual processes and complex toolchains for software development, teams can use a robust CI/CD pipeline to build, test, and deploy software. And AI can streamline the process even further, automatically engineering CI/CD pipelines for consistent quality, compliance, and security checks.\n\nThis guide explains modern CI/CD pipelines, from basic principles to best practices to advanced strategies. You'll also discover how leading organizations use CI/CD for impactful results. What you learn in this guide will help you scale your DevSecOps environment to develop and deliver software in an [agile](https://about.gitlab.com/topics/ci-cd/continuous-integration-agile/), automated, and efficient manner.\n\nWhat you'll learn:\n- [What is continuous integration?](#what-is-continuous-integration%3F)\n- [What is continuous delivery?](#what-is-continuous-delivery%3F)\n- [How source code management relates to CI/CD](#how-source-code-management-relates-to-cicd)\n- [The benefits of CI/CD in modern software development](#the-benefits-of-cicd-in-modern-software-development)\n  - [Key differences between CI/CD and traditional development](#key-differences-between-cicd-and-traditional-development)\n- [Understanding CI/CD fundamentals](#understanding-cicd-fundamentals)\n  - [What is a CI/CD pipeline?](#what-is-a-cicd-pipeline%3F)\n- [Best practices for CI/CD implementation and management](#best-practices-for-cicd-implementation-and-management)\n  - [CI best practices](#ci-best-practices)\n  - [CD best practices](#cd-best-practices)\n- [How to get started with CI/CD](#how-to-get-started-with-cicd)\n- [Security, compliance, and CI/CD](#security-compliance%2C-and-cicd)\n- [CI/CD and the cloud](#cicd-and-the-cloud)\n- [Advanced CI/CD](#advanced-cicd)\n  - [Reuse and automation in CI/CD](#reuse-and-automation-in-cicd)\n  - [Troubleshooting pipelines with AI](#troubleshooting-pipelines-with-ai)\n- [How to migrate to GitLab CI/CD](#how-to-migrate-to-gitlab-cicd)\n- [Lessons from leading organizations](#lessons-from-leading-organizations)\n- [CI/CD tutorials](#cicd-tutorials)\n\n## What is continuous integration?\n\n[Continuous integration](https://about.gitlab.com/topics/ci-cd/benefits-continuous-integration/) (CI) is the practice of integrating all your code changes into the main branch of a shared source code repository early and often, automatically testing changes when you commit or merge them, and automatically kicking off a build. With continuous integration, teams can identify and fix errors and security issues more easily and much earlier in the development process.\n\n## What is continuous delivery?\n[Continuous delivery](https://about.gitlab.com/topics/ci-cd/#what-is-continuous-delivery-cd) (CD) – sometimes called _continuous deployment_ – enables organizations to deploy their applications automatically, allowing more time for developers to focus on monitoring deployment status and assure success. With continuous delivery, DevSecOps teams set the criteria for code releases ahead of time and when those criteria are met and validated, the code is deployed into the production environment. This allows organizations to be more nimble and get new features into the hands of users faster. \n\n## How source code management relates to CI/CD\n\nSource code management ([SCM](https://about.gitlab.com/solutions/source-code-management/)) and CI/CD form the foundation of modern software development practices. SCM systems like [Git](https://about.gitlab.com/blog/what-is-git-the-ultimate-guide-to-gits-role-and-functionality/) provide a centralized way to track changes, manage different versions of code, and facilitate collaboration among team members. When developers work on new features or bug fixes, they create branches from the main codebase, make their changes, and then [merge them through merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/). This branching strategy allows multiple developers to work simultaneously without interfering with each other's code, while maintaining a stable main branch that always contains production-ready code.\n\nCI/CD takes the code managed by SCM systems and automatically builds, tests, and validates it whenever changes are pushed. When a developer submits their code changes, the CI/CD system automatically retrieves the latest code, combines it with the existing codebase, and runs through a series of automated checks. These typically include compiling the code, running unit tests, performing static code analysis, and checking code coverage. If any of these steps fail, the team is immediately notified, allowing them to address issues before they impact other developers or make their way to production. This tight integration between source control and continuous integration creates a feedback loop that helps maintain code quality and prevents integration problems from accumulating.\n\n## The benefits of CI/CD in modern software development\n\n[CI/CD brings transformative benefits to modern software development](https://about.gitlab.com/blog/ten-reasons-why-your-business-needs-ci-cd/) by dramatically reducing the time and risk associated with delivering new features and fixes. The continuous feedback loop gives DevSecOps teams confidence their changes are automatically validated against the entire codebase. The result is higher quality software, faster delivery times, and more frequent releases that can quickly respond to user needs and market demands.\n\nPerhaps most importantly, CI/CD fosters a culture of collaboration and transparency within software development teams. When everyone can see the status of builds, tests, and deployments in real time, it becomes easier to identify and resolve bottlenecks in the delivery process. The automation provided by CI/CD also reduces the cognitive load on developers, freeing them to focus on writing code rather than managing manual deployment processes. This leads to improved developer satisfaction and productivity, while also reducing the risk traditionally associated with the entire software release process. Teams can experiment more freely knowing rapid code reviews are part of the process and they can quickly roll back changes if needed, which encourages innovation and continuous improvement.\n\n> Get started with GitLab CI/CD. [Sign up for GitLab Ultimate](https://about.gitlab.com/free-trial/devsecops/) and try the AI-powered DevSecOps platform free.\n\n### Key differences between CI/CD and traditional development\n\nCI/CD differs from traditional software development in many ways, including:\n\n**Frequent code commits**\n\nDevelopers often work independently and infrequently upload their code to a main codebase, causing merge conflicts and other time-consuming issues. With CI/CD, developers push commits throughout the day, ensuring that conflicts are caught early and the codebase remains up to date.\n\n**Reduced risk**\n\nLengthy testing cycles and extensive pre-release planning are hallmarks of traditional software development. This is done to minimize risk but often hinders the ability to find and fix problems. Risk is managed in CI/CD by applying small, incremental changes that are closely monitored and easily reverted.\n\n**Automated and continuous testing**\n\nIn traditional software development, testing is done once development is complete. However, this causes problems, including delayed delivery and costly bug fixes. CI/CD supports automated testing that occurs continuously throughout development, sparked by each code commit. Developers also receive feedback they can take fast action on.\n\n**Automated, repeatable, and frequent deployments**\n\nWith CI/CD, deployments are automated processes that reduce the typical stress and effort associated with big software rollouts. The same deployment process can be repeated across environments, which saves time and reduces errors and inconsistencies.\n\n## Understanding CI/CD fundamentals\n\nCI/CD serves as a framework for building scalable, maintainable delivery processes, so it's critical for DevSecOps teams to firmly grasp its core concepts. A solid understanding of CI/CD principles enables teams to adapt strategies and practices as technology evolves, rather than being tied to legacy approaches. Here are some of the basics.\n\n### What is a CI/CD pipeline?\n\nA [CI/CD pipeline](https://about.gitlab.com/topics/ci-cd/cicd-pipeline/) is a series of steps, such as build, test, and deploy, that automate and streamline the software delivery process. [Each stage serves as a quality gate](https://about.gitlab.com/blog/guide-to-ci-cd-pipelines/), ensuring that only validated code moves forward. Early stages typically handle basic checks like compilation and unit testing, while later stages may include integration testing, performance testing, compliance testing, and staged deployments to various environments.\n\nThe pipeline can be configured to require manual approvals at critical points, such as before deploying to production, while automating routine tasks and providing quick feedback to developers about the health of their changes. This structured approach ensures consistency, reduces human error, and provides a clear audit trail of how code changes move from development to production. Modern pipelines are often implemented as code, allowing them to be version controlled, tested, and maintained just like application code.\n\nThese are other terms associated with CI/CD that are important to know:\n- **Commit:** a code change\n- **Job:** instructions a runner has to execute\n- **Runner:** an agent or server that executes each job individually that can spin up or down as needed\n- **Stages:** a keyword that defines certain job stages, such as \"build\" and \"deploy.\" Jobs of the same stage are executed in parallel. Pipelines are configured using a version-controlled YAML file, `.gitlab-ci.yml`, at the root level of a project.\n\n![CI/CD pipeline diagram](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749673928/Blog/Content%20Images/1690824533476.png)\n\n## Best practices for CI/CD implementation and management\n\nHow successful you are with CI/CD depends greatly on the [best practices](https://about.gitlab.com/blog/how-to-keep-up-with-ci-cd-best-practices/) you implement. \n\n#### CI best practices\n\n* Commit early, commit often.\n* Optimize pipeline stages.\n* Make builds fast and simple.\n* Use failures to improve processes.\n* Make sure the test environment mirrors production.\n\n#### CD best practices\n\n* Start where you are – you can always iterate.\n* Understand the best continuous delivery is done with minimal tools.\n* Track what’s happening so issues and merge requests don't get out of hand.\n* Streamline user acceptance testing and staging with automation.\n* Manage the release pipeline through automation.\n* Implement monitoring for visibility and efficiency. \n\n> ### Bookmark this!\n>\n>Watch our [\"Intro to CI/CD\" webinar](https://www.youtube.com/watch?v=sQ7Nw3o0izc)!\n>\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n\u003Ciframe src=\"https://www.youtube.com/embed/sQ7Nw3o0izc?si=3HpNqIClrc2ncr7Y\" title=\"Intro to CI/CD webinar\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## How to get started with CI/CD\n\nGetting started with CI/CD begins with identifying a simple but representative project to serve as your pilot. Choose a straightforward application with basic testing requirements, as this allows you to focus on learning the pipeline mechanics rather than dealing with complex deployment scenarios. Begin by ensuring your code is in [version control](https://about.gitlab.com/topics/version-control/) and has some [basic automated tests](https://about.gitlab.com/blog/develop-c-unit-testing-with-catch2-junit-and-gitlab-ci/) — even a few unit tests will suffice. The goal is to [create a minimal pipeline](https://about.gitlab.com/blog/how-to-learn-ci-cd-fast/) that you can gradually enhance as your understanding grows.\n\nFor GitLab specifically, the process starts with creating a `.gitlab-ci.yml` file in your project's root directory. This YAML file defines your pipeline stages (basic ones like build, test, and deploy) and jobs. A simple pipeline might look like this: The build stage compiles your code and creates artifacts, the test stage runs your unit tests, and the deploy stage pushes your application to a staging environment. GitLab will automatically detect this file and start running your pipeline whenever changes are pushed to your repository. The platform provides [built-in runners](https://docs.gitlab.com/runner/) to execute your pipeline jobs, though you can also set up your own runners for more control.\n\nAs you become comfortable with the basics, gradually add more sophisticated elements to your pipeline. This might include adding code quality checks, [security scanning](https://docs.gitlab.com/ee/user/application_security/#security-scanning), or automated deployment to production. GitLab's DevSecOps platform includes features like [compliance management](https://about.gitlab.com/blog/meet-regulatory-standards-with-gitlab/), [deployment variables](https://about.gitlab.com/blog/demystifying-ci-cd-variables/), and manual approval gates that you can incorporate as your pipeline matures. Pay attention to pipeline execution time and look for opportunities to run jobs in parallel where possible. Remember to add proper error handling and notifications so team members are promptly alerted of any pipeline failures. Start documenting common issues and solutions as you encounter them — this will become invaluable as your team grows.\n\n> ### Want to learn more about getting started with CI/CD? Register for a [free CI/CD course on GitLab University](https://university.gitlab.com/courses/continuous-integration-and-delivery-ci-cd-with-gitlab).\n\n## Security, compliance, and CI/CD\n\nOne of the greatest advantages of CI/CD is the ability to embed security and compliance checks early and often in the software development lifecycle. In GitLab, teams can use the `.gitlab-ci.yml` configuration to automatically trigger security scans at multiple stages, from initial code commit to production deployment. The platform's container scanning, dependency scanning, and security scanning capabilities ([Dynamic Application Security Testing](https://docs.gitlab.com/ee/user/application_security/dast/) and [Advanced SAST](https://about.gitlab.com/blog/gitlab-advanced-sast-is-now-generally-available/)) can be configured to run automatically with each code change, checking for vulnerabilities, compliance violations, and security misconfigurations. The platform's API enables integration with [external security tools](https://about.gitlab.com/blog/integrate-external-security-scanners-into-your-devsecops-workflow/), while the test coverage features ensure security tests meet required thresholds.\n\nGitLab's security test reports provide detailed information about findings, enabling quick remediation of security issues before they reach production. The Security Dashboard provides a centralized view of vulnerabilities across projects, while [security policies can be enforced](https://about.gitlab.com/blog/how-gitlab-supports-the-nsa-and-cisa-cicd-security-guidance/) through merge request approvals and pipeline gates. In addition, GitLab provides multiple layers of secrets management to protect sensitive information throughout the CI/CD process, audit logs to track access to secrets, and role-based access control (RBAC) to ensure only authorized users can view or modify sensitive configuration data.\n\nGitLab also supports software bill of materials ([SBOM](https://about.gitlab.com/blog/the-ultimate-guide-to-sboms/)) generation, providing a comprehensive inventory of all software components, dependencies, and licenses in an application and enabling teams to quickly identify and respond to vulnerabilities and comply with regulatory mandates.\n\n## CI/CD and the cloud\n\nGitLab's CI/CD platform provides robust integration with major cloud providers including [Amazon Web Services](https://about.gitlab.com/partners/technology-partners/aws/), [Google Cloud Platform](https://about.gitlab.com/blog/provision-group-runners-with-google-cloud-platform-and-gitlab-ci/), and [Microsoft Azure](https://docs.gitlab.com/ee/install/azure/), enabling teams to automate their cloud deployments directly from their pipelines. Through GitLab's cloud integrations, teams can manage cloud resources, deploy applications, and monitor cloud services all within the GitLab interface. The platform's built-in cloud deployment templates and [Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/) features significantly reduce the complexity of cloud deployments, allowing teams to focus on application development rather than infrastructure management. For organizations that want to automate their IT   infrastructure using GitOps, GitLab has a [Flux CD integration](https://about.gitlab.com/blog/why-did-we-choose-to-integrate-fluxcd-with-gitlab/).\n\nGitLab's cloud capabilities extend beyond basic deployment automation. The platform's [Kubernetes integration](https://about.gitlab.com/blog/kubernetes-overview-operate-cluster-data-on-the-frontend/) enables teams to manage container orchestration across multiple cloud providers, while the [cloud native GitLab installation options](https://about.gitlab.com/topics/ci-cd/cloud-native-continuous-integration/) allow the platform itself to run in cloud environments. Through GitLab's cloud-native features, teams can implement auto-scaling runners that dynamically provision cloud resources for pipeline execution, optimizing costs and performance. The platform's integration with cloud provider security services ensures that security and compliance requirements are met throughout the deployment process.\n\nFor multi-cloud environments, GitLab provides consistent workflows and tooling regardless of the underlying cloud provider. Teams can use GitLab's environment management features to handle different cloud configurations across development, staging, and production environments. The platform's [infrastructure as code](https://docs.gitlab.com/ee/user/infrastructure/iac/) support, particularly its native integration with Terraform, enables teams to version control and automate their cloud infrastructure provisioning. GitLab's monitoring and observability features integrate with cloud provider metrics, providing comprehensive visibility into application and infrastructure health across cloud environments.\n\n## Advanced CI/CD \nCI/CD has evolved far beyond simple build and deploy pipelines. In advanced implementations, CI/CD involves sophisticated orchestration of automated testing, security scanning, infrastructure provisioning, AI, and more. Here are a few advanced CI/CD strategies that can help engineering teams scale their pipelines and troubleshoot issues even as architectural complexity grows.\n\n### Reuse and automation in CI/CD\n\nGitLab is transforming how development teams create and manage CI/CD pipelines with two major innovations: the [CI/CD Catalog](https://about.gitlab.com/blog/ci-cd-catalog-goes-ga-no-more-building-pipelines-from-scratch/) and [CI/CD steps](https://about.gitlab.com/blog/introducing-ci-cd-steps-a-programming-language-for-devsecops-automation/), a new programming language for DevSecOps automation currently in experimental phase. The CI/CD Catalog is a centralized platform where developers can discover, reuse, and contribute CI/CD components. Components function as reusable, single-purpose building blocks that simplify pipeline configuration — similar to Lego pieces for CI/CD workflows. Meanwhile, CI/CD steps support complex workflows by allowing developers to compose inputs and outputs for a CI/CD job. With the CI/CD Catalog and CI/CD steps, DevSecOps teams can easily standardize CI/CD and its components, simplifying the process of developing and maintaining CI/CD pipelines.\n\n> Learn more in our [CI/CD Catalog FAQ](https://about.gitlab.com/blog/faq-gitlab-ci-cd-catalog/) and [CI/CD steps documentation](https://docs.gitlab.com/ee/ci/steps/).\n\n### Troubleshooting pipelines with AI\n\nWhile CI/CD pipelines can and do break, troubleshooting the issue quickly can minimize the impact. GitLab Duo Root Cause Analysis, part of a suite of AI-powered features, removes the guesswork by [determining the root cause for a failed CI/CD pipeline](https://about.gitlab.com/blog/quickly-resolve-broken-ci-cd-pipelines-with-ai/). When a pipeline fails, GitLab provides detailed job logs, error messages, and execution traces that show exactly where and why the failure occurred. Root Cause Analysis then uses AI to suggest a fix.\nWatch GitLab Duo Root Cause Analysis in action:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n\u003Ciframe src=\"https://www.youtube.com/embed/sTpSLwX5DIs?si=J6-0Bf6PtYjrHX1K\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n## How to migrate to GitLab CI/CD\n\nMigrating to the DevSecOps platform and its built-in CI/CD involves a systematic approach of analyzing your existing pipeline configurations, dependencies, and deployment processes to map them to GitLab's equivalent features and syntax. Use these guides to help make the move.\n\n* [How to migrate from Bamboo to GitLab CI/CD](https://about.gitlab.com/blog/migrating-from-bamboo-to-gitlab-cicd/)\n* [Jenkins to GitLab: The ultimate guide to modernizing your CI/CD environment](https://about.gitlab.com/blog/jenkins-gitlab-ultimate-guide-to-modernizing-cicd-environment/)\n* [GitHub to GitLab migration the easy way](https://about.gitlab.com/blog/github-to-gitlab-migration-made-easy/)\n\n## Lessons from leading organizations\n\nThese leading organizations migrated to GitLab and are enjoying the myriad benefits of CI/CD. Read their stories.\n\n- [Lockheed Martin](https://about.gitlab.com/customers/lockheed-martin/)\n- [Indeed](https://about.gitlab.com/blog/how-indeed-transformed-its-ci-platform-with-gitlab/)\n- [CARFAX](https://about.gitlab.com/customers/carfax/)\n- [HackerOne](https://about.gitlab.com/customers/hackerone/)\n- [Betstudios](https://about.gitlab.com/blog/betstudios-cto-on-improving-ci-cd-capabilities-with-gitlab-premium/)\n- [Thales and Carrefour](https://about.gitlab.com/blog/how-carrefour-and-thales-are-evolving-their-ci-cd-platforms/)\n\n## CI/CD tutorials\n\nBecome a CI/CD expert with these easy-to-follow tutorials.\n\n* [Basics of CI: How to run jobs sequentially, in parallel, or out of order](https://about.gitlab.com/blog/basics-of-gitlab-ci-updated/)\n* [How to set up your first GitLab CI/CD component](https://about.gitlab.com/blog/tutorial-how-to-set-up-your-first-gitlab-ci-cd-component/)\n* [Building a GitLab CI/CD pipeline for a monorepo the easy way](https://about.gitlab.com/blog/building-a-gitlab-ci-cd-pipeline-for-a-monorepo-the-easy-way/)\n* [Using child pipelines to continuously deploy to five environments](https://about.gitlab.com/blog/using-child-pipelines-to-continuously-deploy-to-five-environments/)\n* [CI/CD automation: Maximize 'deploy freeze' impact across GitLab groups](https://about.gitlab.com/blog/ci-cd-automation-maximize-deploy-freeze-impact-across-gitlab-groups/)\n* [Refactoring a CI/CD template to a CI/CD component](https://about.gitlab.com/blog/refactoring-a-ci-cd-template-to-a-ci-cd-component/)\n* [Annotate container images with build provenance using Cosign in GitLab CI/CD](https://about.gitlab.com/blog/annotate-container-images-with-build-provenance-using-cosign-in-gitlab-ci-cd)\n\n> #### Get started with GitLab CI/CD. [Sign up for GitLab Ultimate](https://about.gitlab.com/free-trial/devsecops/) and try the AI-powered DevSecOps platform.",[108,851,495,9,830,694],{"slug":4475,"featured":90,"template":700},"ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation","content:en-us:blog:ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation.yml","Ultimate Guide To Ci Cd Fundamentals To Advanced Implementation","en-us/blog/ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation.yml","en-us/blog/ultimate-guide-to-ci-cd-fundamentals-to-advanced-implementation",{"_path":4481,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4482,"content":4488,"config":4492,"_id":4494,"_type":13,"title":4495,"_source":15,"_file":4496,"_stem":4497,"_extension":18},"/en-us/blog/ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab",{"title":4483,"description":4484,"ogTitle":4483,"ogDescription":4484,"noIndex":6,"ogImage":4485,"ogUrl":4486,"ogSiteName":685,"ogType":686,"canonicalUrls":4486,"schema":4487},"Ultimate guide to migrating from AWS CodeCommit to GitLab","Learn how to migrate from AWS Services to GitLab and seamlessly integrate with the DevSecOps platform in this comprehensive tutorial.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097810/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2828%29_4mi0l4wzUa5VI4wtf8gInx_1750097810027.png","https://about.gitlab.com/blog/ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Ultimate guide to migrating from AWS CodeCommit to GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tsukasa Komatsubara\"},{\"@type\":\"Person\",\"name\":\"Darwin Sanoy\"},{\"@type\":\"Person\",\"name\":\"Samer Akkoub\"},{\"@type\":\"Person\",\"name\":\"Bart Zhang\"}],\n        \"datePublished\": \"2024-08-26\",\n      }",{"title":4483,"description":4484,"authors":4489,"heroImage":4485,"date":2538,"body":4490,"category":694,"tags":4491},[4416,691,4396,3931],"On July 25, 2024, AWS made a significant announcement regarding its CodeCommit service. As detailed in their [official blog post](https://aws.amazon.com/blogs/devops/how-to-migrate-your-aws-codecommit-repository-to-another-git-provider/), AWS has decided to close new customer access to CodeCommit. While existing customers can continue using the service, AWS will not introduce new features, focusing only on security, availability, and performance improvements.\n\nThis announcement has prompted development teams to consider migrating their repositories to alternative Git providers. In light of these changes, we've prepared this comprehensive guide to assist teams in migrating to GitLab and integrating with other AWS services.\n\n**Note:** For more details on AWS's official migration recommendations, please refer to [their blog post](https://aws.amazon.com/blogs/devops/how-to-migrate-your-aws-codecommit-repository-to-another-git-provider/).\n\n## About this guide\n\nThis guide provides comprehensive information for development teams using GitLab who are considering integration with AWS services or planning to migrate from AWS-hosted Git repositories to GitLab.com. The guide is structured into three main sections:\n\n- [Parallel migration to GitLab](#section-1-parallel-migration-to-gitlab): Explains how to gradually migrate from existing AWS-hosted repositories to GitLab.com while minimizing risks.\n\n- [Integration with AWS CodeBuild](#section-2-integrating-gitlab-with-aws-codebuild): Provides steps to integrate GitLab repositories with AWS CodeBuild, setting up a powerful continuous integration (CI) environment.\n\n- [Integration with AWS CodePipeline](#section-3-integrating-gitlab-with-aws-codepipeline): Details how to connect GitLab repositories with AWS CodePipeline to build efficient continuous delivery (CD) pipelines.\n\n- [Downstream integrations for CodePipeline and CodeStar Connections](#section-4-migrating-to-gitlab): Explains how to leverage GitLab-AWS connections for widespread service access, unlocking a cascade of integration possibilities across the AWS ecosystem.\n\nThrough this guide, you'll learn how to combine the powerful features of GitLab and AWS to create an efficient and flexible development workflow.\n\n## Section 1: Parallel migration to GitLab \n\nFor those considering migrating Git repositories hosted on AWS to GitLab.com, this section, which is a phased approach, introduces methods to achieve migration while minimizing risks. By leveraging GitLab's mirroring capabilities, you can maintain existing development flows while testing the new environment.\n\n### Why is parallel migration important?\n\nLarge-scale system migrations always involve risks, particularly potential impacts on ongoing development work, existing integrations, and automated processes. Adopting a parallel migration approach offers the following benefits:\n\n1. Risk minimization: Test the new environment while keeping existing systems operational.\n2. Seamless transition: Development teams can gradually acclimate to the new system.\n3. Integration testing: Thoroughly test all integrations and automation in the new environment.\n4. Future-proofing: Enable teams to gradually migrate to GitLab CI/CD in parallel to existing CI.\n\nParallel migration is not required if it is already known that you want to cut over directly to GitLab.\n\n### Steps for migrating to GitLab.com\n\n#### Step 1: Get set up on GitLab.com\n\n- Check if your company already has a group in use on GitLab.com and whether they have single sign-on (SSO) set up – if they do, then you will want to use both.\n\n- If your company does not have a presence on GitLab.com, visit [GitLab.com](www.gitlab.com) and create a new account or log in to an existing one.\n- Create a new company namespace (a group at the root level of gitlab.com).\n- Pick a name that reflects your entire company (and is not already taken).\n\n#### Step 2: Import repository\nFor parallel migration: Use GitLab's pull mirroring feature to automatically sync changes from AWS-hosted repositories to GitLab.com.\n\n1. Navigate to the target group GitLab.com.\n2. In the upper right, click \"New project.\"\n3. On the \"Create new project\" page, click \"Import project.\"\n4. On the \"Import project\" page, click \"Repository by URL.\"\n5. Enter the URL of your AWS-hosted repository in the \"Git repository URL\" field.\n6. Underneath the Git repository URL field, check \"Mirror repository.\"\n7. Set up authentication: in the AWS CodeCommit console, select the clone URL for the repository you will migrate. If you plan on importing CodeCommit repositories into GitLab, you can use the HTTPS CodeCommit URL to clone the repository via GitLab Repository Mirroring. You will need to also provide your Git credentials from AWS for your identity and access management (IAM) user within GitLab. You can create Git credentials for AWS CodeCommit by following this [AWS guide](https://docs.aws.amazon.com/codecommit/latest/userguide/setting-up-gc.html).\n\n![Clone URL](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/clone-url-screenshot__1__aHR0cHM6_1750097822121.png)\n\nThis setup will automatically pull changes from the AWS-hosted repository to GitLab.com every five minutes by default.\n\nFor more information, read our [repository mirroring documentation](https://docs.gitlab.com/ee/user/project/repository/mirror/).\n\n#### Step 3: Test and validate integrations\n\n1. CI/CD pipelines: Set up the `.gitlab-ci.yml` file in GitLab CI to replicate existing pipelines. You can read more about [planning a migration from other CI tools into GitLab CI/CD](https://docs.gitlab.com/ee/ci/migration/plan_a_migration.html).\n2. Issue tracking: Import project issues and test workflows.\n3. Code review: Set up the merge request process and test review workflows.\n\n#### Step 4: Gradual migration\n\n1. Start with small or non-critical projects to familiarize yourself with working on GitLab.com.\n2. Provide training for team members and allow time to adapt to new workflows.\n3. Gradually migrate more projects while ensuring integrations and workflows are problem-free.\n\nFor more information, see [Automating Migrations from CodeCommit to GitLab](https://gitlab.com/guided-explorations/aws/migrating-from-codecommit-to-gitlab/-/blob/main/migrating_codecommit_to_gitlab.md).\n\n#### Step 5: Complete migration\nOnce all tests and validations are complete and the team is comfortable with the new environment, plan for full migration. For each project:\n\n1. Set a migration date and notify all stakeholders.\n2. Perform final data synchronization.\n3. Remove mirroring settings from the GitLab project.\n4. Set AWS-hosted repositories to read-only and transition all development work to GitLab.com.\n\n#### Step 6: Assess adoption of new capabilities\n\nGitLab collaboration and workflow automation for developers is far richer than CodeCommit. It merits some time to learn what these capabilities are. The merge request process is especially rich compared to CodeCommit.\n\nAfter repositories are stable on GitLab, it is very easy to experiment with GitLab CI/CD in parallel to an existing solution. Teams can take time to perfect their GitLab CI/CD automation while production workflows remain unaffected.\n\nGitLab artifact management is also very capable with the Releases feature and many package registries.\n\n### Section 1: Summary\nBy adopting a parallel migration approach to GitLab, you can achieve a smooth transition while minimizing risks. This process allows teams to gradually adapt to the new environment and ensure all integrations and automations function correctly. Cutover migrations only omit a single setting checkbox if it is known that a parallel migration is not necessary.\n\n## Section 2: Integrating GitLab with AWS CodeBuild\n\nFor those wanting to build and test code from GitLab repositories using AWS CodeBuild, this comprehensive guide will help you set up an efficient CI pipeline.\n\n### Prerequisites\n\n- GitLab.com account\n- AWS account\n- AWS CLI (configured)\n\n### Step 1: Create GitLab connection in AWS CodeStar Connections\n\n1. Log in to the AWS Management Console and navigate to the CodeBuild service.\n2. Select \"Settings\" > \"Connections\" from the left navigation panel.\n3. Click the \"Create connection\" button.\n4. Choose \"GitLab\" as the provider.\n5. Enter a connection name and click \"Connect to GitLab.\"\n6. You'll be redirected to the GitLab authentication page.\n7. Approve the necessary permissions.\n8. Once successful, the connection status will change to \"Available.\"\n\n![CodeStar Connect setup](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/codestar-connections-setup_aHR0cHM6_1750097822122.png)\n\n### Step 2: Create AWS CodeBuild project\n\n1. Click \"Create build project\" on the CodeBuild dashboard.\n2. Enter a project name and description.\n3. For source settings, select \"GitLab\" as the provider.\n4. Choose the connection you just created and specify the GitLab repository and branch.\n\n![Add CodeBuild project](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/codepipeline_step_3_add_codebuild_aHR0cHM6_1750097822123.png)\n\n**Note: From Step 3 forward, please configure the settings according to your specific environment and needs.**\n\n### Summary of Section 2\nThis section explained in detail how to integrate GitLab repositories with AWS CodeBuild. This setup enables a continuous integration pipeline where code changes in GitLab are automatically built and tested using AWS CodeBuild.\n\n## Section 3: Integrating GitLab with AWS CodePipeline\n\nFor those looking to implement continuous delivery from GitLab repositories using AWS CodePipeline, this detailed guide will be helpful. The integration has become even easier now that GitLab is available as an AWS CodeStar Connections provider.\n\n### Prerequisites\n\n- GitLab.com account\n- AWS account\n- AWS CLI (configured)\n\n### Step 1: Create GitLab connection in AWS CodeStar Connections\n\n1. Log in to the AWS Management Console and navigate to the CodePipeline service.\n2. Select \"Settings\" > \"Connections\" from the left navigation panel.\n3. Click the \"Create connection\" button.\n4. Choose \"GitLab\" as the provider.\n5. Enter a connection name and click \"Connect to GitLab.\"\n6. You'll be redirected to the GitLab authentication page.\n7. Approve the necessary permissions.\n8. Once successful, the connection status will change to \"Available.\"\n\n![CodeStar Connections setup](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/codestar-connections-setup_aHR0cHM6_1750097822125.png)\n\n### Step 2: Create AWS CodePipeline\n\n1. Click \"Create pipeline\" on the CodePipeline dashboard.\n2. Enter a pipeline name and click \"Next.\"\n3. Select \"GitLab\" as the source provider.\n4. Choose the connection you just created and specify the GitLab repository and branch.\n5. Select the Trigger type: You can trigger CodePipeline pipeline execution based on either pull or push events against specific branches and file types within your repository.\n\n![Add source provider](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/codepipeline_step_2_source_provider_aHR0cHM6_1750097822127.png)\n\n![Add source configuration](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/codepipeline_step_2_source_configured_aHR0cHM6_1750097822129.png)\n\n**Note: From Step 3 forward, please configure the settings according to your specific environment and needs.**\n\n### Summary of Section 3\nThis section detailed how to integrate GitLab repositories with AWS CodePipeline. This setup enables a continuous delivery pipeline where code changes in GitLab are automatically deployed to your AWS environment.\n\n## Section 4: Migrating to GitLab\n\nIntegrating GitLab with AWS unlocks powerful capabilities for streamlining your development and deployment workflows and helps to solve your source code management woes. This integration can be achieved in several ways, each offering unique benefits:\n\n- Using AWS CodeStar Connections to link GitLab with AWS services enables a more cohesive workflow by allowing external Git repositories, like GitLab, to connect with various AWS services. This setup supports automated builds, deployments, and other essential actions directly from your GitLab repository, making your development process more integrated and streamlined.\n\n- Connecting GitLab with AWS CodePipeline via AWS CodeStar Connections takes automation to the next level by allowing you to create a full CI/CD pipeline. This approach integrates GitLab with AWS CodePipeline, enabling you to automate the entire process – from source control and builds to testing and deployment – using AWS services like CodeBuild and CodeDeploy. This ensures a robust, scalable, and efficient delivery process.\n\n![Chart of new technology and solutions for using GitLab and AWS together](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097822/Blog/Content%20Images/Blog/Content%20Images/Announcing_New_Technology_and_Solutions_for_using_GitLab_and_AWS_Together_aHR0cHM6_1750097822130.png)\n\n1\\. Connecting GitLab with AWS services using AWS CodeStar Connections\n\nAWS CodeStar Connections is a service that allows you to connect external Git repositories (such as GitHub or Bitbucket) to AWS services. You can also connect GitLab to AWS services via CodeStar Connections. When using GitLab, you may need to set up a custom connection as an HTTP Git server.\nThe following AWS services can be connected to GitLab using this method:\n\n- **AWS Service Catalog**\n\nAWS Service Catalog helps organizations standardize and manage AWS resources. Integrating it with GitLab improves transparency in resource management and simplifies change tracking. Specifically, you can automate catalog updates based on GitLab commits, enhancing operational efficiency.\n\n- __AWS CodeBuild__\n\nAWS CodeBuild is a managed build service that compiles source code, runs tests, and produces deployable software packages. Integrating GitLab with CodeBuild allows automated build processes to start whenever code changes are pushed to GitLab. This ensures consistency in builds and facilitates easier collaboration and version control.\n\n- __AWS Glue Notebook Jobs__\n\nAWS Glue Notebook Jobs is a service that allows you to interactively develop and run data preparation and ETL (Extract, Transform, Load) tasks. Integrating GitLab with Glue Notebook Jobs enables version control for notebooks and ETL scripts, promotes collaboration among team members, and improves the quality management of data processing pipelines.\n\n- __AWS Proton__\n\nAWS Proton is a service that automates the development and deployment of microservices and serverless applications. By integrating GitLab with AWS Proton, you can manage infrastructure as code, automate deployments, and ensure consistent environment management, leading to more efficient development processes.\n\nAs AWS CodeStar Connections supports more services, connecting GitLab with additional AWS services will become easier. It's advisable to regularly check for new services that support CodeStar Connections.\n\n2. Connecting CodePipeline with GitLab via AWS CodeStar Connections (including CodeDeploy)\n\nAWS CodePipeline is a continuous delivery service that automates the release process for software. To connect GitLab with CodePipeline, you need to use AWS CodeStar Connections. This setup allows you to designate a GitLab repository as the source and automate the entire CI/CD pipeline.\nThe primary actions supported by CodePipeline include:\n- **Source control:** AWS CodeCommit, GitHub, Bitbucket, GitLab\n- **Build and test:** AWS CodeBuild, Jenkins\n- **Deploy:** AWS CodeDeploy, Elastic Beanstalk, ECS, S3\n- **Approval:** Manual approval\n- **Infrastructure management:** AWS CloudFormation\n- **Serverless:** AWS Lambda\n- **Testing:** AWS Device Farm\n- **Custom Actions:** AWS Step Functions\n\nBy integrating GitLab with CodePipeline, you can automatically trigger the pipeline whenever code changes are pushed to GitLab, allowing a consistent process from build to deployment. Additionally, combining this with GitLab's version control capabilities makes it easier to track deployment history and states, leading to more flexible and reliable software delivery.\n\n## What you've learned\nThis guide has provided comprehensive information on migrating to and integrating GitLab with AWS. Through the four main topics, we've covered:\n- Parallel migration to GitLab: How to gradually migrate from existing AWS-hosted repositories to GitLab.com while minimizing risks.\n- Integration with AWS CodeBuild: Steps to set up a powerful CI environment integrated with GitLab repositories.\n- Integration with AWS CodePipeline: How to build efficient continuous delivery pipelines using GitLab repositories.\n- Downstream integrations for CodePipeline and CodeStar Connections: Leveraging GitLab-AWS connections for widespread service access, unlocking a cascade of integration possibilities across the AWS ecosystem.\n\nAs every organization's code hosting and integration implementation strategy is unique, this tutorial may be used as a starting point for your own GitLab + AWS integration and implementation strategy.\n\n## Additional resources\n\nFor more detailed information and advanced configurations, refer to the following resources:\n\n- [GitLab documentation](https://docs.gitlab.com/)\n- [AWS CodeBuild User Guide](https://docs.aws.amazon.com/codebuild/latest/userguide/welcome.html)\n- [AWS CodePipeline User Guide](https://docs.aws.amazon.com/codepipeline/latest/userguide/welcome.html)\n- [GitLab CI/CD documentation](https://docs.gitlab.com/ee/ci/)\n- [Integrate with AWS](https://docs.gitlab.com/ee/solutions/cloud/aws/gitlab_aws_integration.html)\n\nIf you have questions or need support, please contact [GitLab Support](https://about.gitlab.com/support/) or AWS Support. We hope this comprehensive guide helps you in your AWS-GitLab integration journey.",[108,810,495,9,1016,694,232],{"slug":4493,"featured":90,"template":700},"ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab","content:en-us:blog:ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab.yml","Ultimate Guide To Migrating From Aws Codecommit To Gitlab","en-us/blog/ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab.yml","en-us/blog/ultimate-guide-to-migrating-from-aws-codecommit-to-gitlab",{"_path":4499,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4500,"content":4506,"config":4513,"_id":4515,"_type":13,"title":4516,"_source":15,"_file":4517,"_stem":4518,"_extension":18},"/en-us/blog/understanding-and-improving-total-blocking-time",{"title":4501,"description":4502,"ogTitle":4501,"ogDescription":4502,"noIndex":6,"ogImage":4503,"ogUrl":4504,"ogSiteName":685,"ogType":686,"canonicalUrls":4504,"schema":4505},"Total Blocking Time - The metric to know for faster website performance","Learn how to identify and fix some root causes for high Total Blocking Time.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749682637/Blog/Hero%20Images/tbt_cover_image.jpg","https://about.gitlab.com/blog/understanding-and-improving-total-blocking-time","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Total Blocking Time - The metric to know for faster website performance\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Jacques Erasmus\"}],\n        \"datePublished\": \"2023-02-14\",\n      }",{"title":4501,"description":4502,"authors":4507,"heroImage":4503,"date":4509,"body":4510,"category":785,"tags":4511},[4508],"Jacques Erasmus","2023-02-14","\n\nOur world overwhelms us with information that is more accessible than ever. The increasing rates of content production and consumption are gifts that keep on giving. We can't seem to keep up with the information thrown at us. We're limited by our cognitive limitations and time constraints, and a [recent study](https://www.nature.com/articles/s41467-019-09311-w) concluded the result is a shortening of attention spans. Websites are no exception.\n\nUsers who interact with your website want feedback, and want it fast. Preferably immediately! Website performance has become an important factor in keeping users engaged. But how do you measure how unresponsive a page is before it becomes fully interactive?\n\nMany [performance metrics](https://web.dev/vitals/) exist, but this blog post focuses on Total Blocking Time (TBT).\n\n## What is Total Blocking Time?\n\nTBT measures the total amount of time tasks were blocking your browser's main thread. This metric represents the total amount of time that a user could not interact with your website. It's measured between [First Contentful Paint (FCP)](https://web.dev/fcp/) and [Time to Interactive (TTI)](https://web.dev/tti/), and represents the combined blocking time for all long tasks.\n\n## What is a long task?\n\nA long task is a process that runs on the main thread for longer than 50 milliseconds (ms). After a task starts, a browser can't interrupt it, and a single long-running task can block the main thread. The result: a website that is unresponsive to user input until the task completes.\n\nAfter the first 50 ms, all time spent on a task is counted as _blocking time_. This diagram shows five tasks, two of which block the main thread for 140 ms:\n\n![A diagram containing five tasks, two of which are blocking the main thread. The TBT for these tasks adds up to 140 ms.](https://about.gitlab.com/images/blogimages/tbt/tasks_diagram.png)\n\n## How can we measure TBT?\n\nMany tools measure TBT, but here we’ll use [Chrome DevTools](https://developer.chrome.com/docs/devtools/evaluate-performance/) to analyze runtime performance.\n\nAs an example: We recently improved performance on GitLab's [**View Source** page](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab-ci.yml). This screenshot, taken before the performance improvement, shows eight long-running tasks containing a TBT of **2388.16 ms**. That's more than **two seconds**:\n\n![A screenshot indicating that there are eight long-running tasks. The TBT of these tasks adds up to 2388.16 ms.](https://about.gitlab.com/images/blogimages/tbt/summary_before.png)\n\n## How can we improve TBT?\n\nAs you might have guessed by now, reducing the time needed to complete long-running tasks reduces TBT.\n\nBy selecting one of the tasks from the previous screenshot, we can get a breakdown of how the browser executed it. This **Bottom-Up** view shows that much time is spent on rendering content in the Document Object Model (DOM):\n\n![A screenshot of the Bottom-Up view of one of tasks from the previous screenshot. It indicates that most of the time is being spent on rendering content in the DOM.](https://about.gitlab.com/images/blogimages/tbt/task_7_before.png)\n\nThis page has a lot of content that is below the fold – not immediately visible. The browser is spending a lot of resources upfront to render content that is not even visible to the user yet!\n\nSo what can we do? Some ideas:\n\n- **Change the UX.**\n  - Add a Show More button, paging, or virtual scrolling for long lists.\n- **Lazy-load images.**\n  ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65745))\n    - Lazy-loading images reduces page weight, allowing the browser to spend resources on more important tasks.\n- **Lazy-load long lists.**\n  ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71633))\n    - Similar to lazy-loading images, this approach allows the browser to spend resources on more important tasks.\n- **Reduce excessive HTML.**\n  ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65835))\n    - For example, when loading large pages consider removing unnecessary content. Or, consider rendering some content (like icons) with CSS instead.\n- **Defer rendering when possible.**\n    - The [`content-visibility: auto;`](https://developer.mozilla.org/en-US/docs/Web/CSS/content-visibility) CSS property ensures the rendering of off-screen elements (and thus irrelevant to the user) is skipped without affecting the page layout. ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67050))\n    - The [Intersection Observer API](https://developer.mozilla.org/en-US/docs/Web/API/Intersection_Observer_API) allows you to observe when elements intersect with the viewport. This information can be used to show or hide certain elements. ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71633))\n    - The global [`requestIdleCallback` method](https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback?qs=requestIdleCallback) can be used to render content after the browser goes into an idle state.\n  ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101942/diffs#7eed73783787184e5b1c029b9668e48638f3a6e8_64_78))\n\nFrameworks such as VueJS and React are already heavily optimized. However, be mindful of how you use these frameworks to avoid expensive tasks.\n\n### Change VueJS usage to improve TBT\n\nThis screenshot shows the **Bottom-Up** view of a task. Much of the task time is spent on activities from third-party code in the VueJS framework:\n\n![A screenshot of the Bottom-Up view of one of tasks. It indicates that a lot of the time is being spent on activities in the third-party VueJS framework.](https://about.gitlab.com/images/blogimages/tbt/task_6_before.png)\n\nWhat improvements can we make?\n\n- **Use [Server-side rendering (SSR)](https://gitlab.com/gitlab-org/gitlab/-/issues/215365) or [streaming](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/101)** for pages that are sensitive to page load performance.\n- **If you don't _need_ Vue, don't use it.**\n  Component instances are a lot more expensive than using plain DOM nodes. Try to avoid unnecessary component abstractions.\n- **Optimize component [props](https://vuejs.org/guide/components/props.html).**\n  Child components in Vue update when at least one of their received props are being updated. Analyze the data that you pass to components. You may find that you can avoid unnecessary updates by making changes to your props strategy.\n- **Use [v-memo](https://vuejs.org/api/built-in-directives.html#v-memo) to skip updates.**\n    - In Vue versions 3.2 and later, `v-memo` enables you to cache parts of your template. The cached template updates and re-renders only if one of its provided dependencies changes.\n- **Use [v-once](https://vuejs.org/api/built-in-directives.html#v-once) for data** that does not need to be reactive after the initial load.\n  ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101942))\n    - `v-once` ensures the element and component are only rendered once. Any future updates will be skipped.\n- **Reduce expensive tasks in your Vue components.**\n  Even a small script may take a long time to finish if it’s not optimized enough. Some suggestions:\n    - By using [`requestIdleCallback`](https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback?qs=requestIdleCallback) you can defer the execution of the non-critical tasks. ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101942/diffs#7eed73783787184e5b1c029b9668e48638f3a6e8_64_78))\n    - By executing expensive scripts in [WebWorkers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) you can unblock the main thread.\n\n### Results and methods\n\nBy using three of the methods suggested above, we reduced TBT from about **3 seconds** to approximately **500 ms**:\n\n![A chart indicating a drop in TBT from ~3 seconds to ~500 milliseconds.](https://about.gitlab.com/images/blogimages/tbt/chart_after.png)\n\nWhat did we do?\n\n- Deferred rendering by using the [`content-visibility: auto;`](https://developer.mozilla.org/en-US/docs/Web/CSS/content-visibility) CSS property.\n- Deferred rendering by using the [Intersection Observer API](https://developer.mozilla.org/en-US/docs/Web/API/Intersection_Observer_API).\n- Used [v-once](https://vuejs.org/api/built-in-directives.html#v-once) for content that didn't need to be reactive after rendering.\n\nRemember, the size of the decrease always depends on how optimized your app already is to begin with.\n\nThere is a lot more we can do to improve TBT. While the specific approach depends on the app you're optimizing, the general methods discussed here are very effective at finding improvement opportunities in any app. Like most things in life, a series of the smallest changes often yield the biggest impact. So let's [iterate](/blog/dont-confuse-these-twelve-shortcuts-with-iteration/) together, and adapt to this ever-changing world.\n\n> “Adaptability is the simple secret of survival.” – Jessica Hagedorn\n\n_Cover image by [Growtika](https://unsplash.com/@growtika?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText) on [Unsplash](https://unsplash.com/photos/Iqi0Rm6gBkQ?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)_\n",[4512,744,9],"frontend",{"slug":4514,"featured":6,"template":700},"understanding-and-improving-total-blocking-time","content:en-us:blog:understanding-and-improving-total-blocking-time.yml","Understanding And Improving Total Blocking Time","en-us/blog/understanding-and-improving-total-blocking-time.yml","en-us/blog/understanding-and-improving-total-blocking-time",{"_path":4520,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4521,"content":4527,"config":4531,"_id":4533,"_type":13,"title":4534,"_source":15,"_file":4535,"_stem":4536,"_extension":18},"/en-us/blog/use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project",{"title":4522,"description":4523,"ogTitle":4522,"ogDescription":4523,"noIndex":6,"ogImage":4524,"ogUrl":4525,"ogSiteName":685,"ogType":686,"canonicalUrls":4525,"schema":4526},"Use GitLab Duo to build and deploy a simple Quarkus-native project","This tutorial shows how a Java application is compiled to machine code and deployed to a Kubernetes cluster using a CI/CD pipeline. See how AI makes the process faster and more efficient.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666069/Blog/Hero%20Images/AdobeStock_639935439.jpg","https://about.gitlab.com/blog/use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Use GitLab Duo to build and deploy a simple Quarkus-native project\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2024-10-17\",\n      }",{"title":4522,"description":4523,"authors":4528,"heroImage":4524,"date":3536,"body":4529,"category":718,"tags":4530},[715],"In [“How to automate software delivery using Quarkus and\nGitLab,”](https://about.gitlab.com/blog/how-to-automate-software-delivery-using-quarkus-and-gitlab/)\nyou learned how to develop and deploy a simple Quarkus-JVM application to a\nKubernetes cluster using [GitLab Auto\nDevOps](https://docs.gitlab.com/ee/topics/autodevops/). Now, you'll learn\nhow to use Quarkus-native to compile a Java application to machine code and\ndeploy it to a Kubernetes cluster using a CI/CD pipeline. Follow our journey\nfrom development to deployment leveraging [GitLab\nDuo](https://about.gitlab.com/gitlab-duo/) as our AI companion, including\nthe specific prompts we used.\n\n\n## What is Quarkus?\n\n\n[Quarkus](https://quarkus.io/), also known as the Supersonic Subatomic Java,\nis an open source, Kubernetes-native Java stack tailored to OpenJDK HotSpot\nand GraalVM. The Quarkus project recently moved to the [Commonhaus\nFoundation](https://www.commonhaus.org/), a nonprofit organization dedicated\nto the sustainability of open source libraries and frameworks that provides\na balanced approach to governance and support.\n\n\n## Prerequisites\n\n\nThis tutorial assumes:\n\n\n- You have a running Kubernetes cluster, e.g. GKE.\n\n- You have access to the Kubernetes cluster from your local laptop via the\n`kubectl` command.\n\n- The cluster is connected to your GitLab project.\n\n- You have [Maven (Version 3.9.6 or later)](https://maven.apache.org/)\ninstalled on your local laptop.\n\n- You have Visual Studio Code installed on your local laptop.\n\n\nIf you’d like to set up a Kubernetes cluster connected to your GitLab\nproject, you can follow the instructions in this\n[tutorial](https://about.gitlab.com/blog/eliminate-risk-with-feature-flags-tutorial/),\nup to but not including the “Creating an instance of MySQL database in your\ncluster via Flux” section (you do not need a database for this tutorial).\n\n\nYou will also need to install an nginx ingress in your Kubernetes cluster.\nHere are two ways to do this:\n\n1. You can follow the instructions in [“Creating and importing\nprojects”](https://about.gitlab.com/blog/eliminate-risk-with-feature-flags-tutorial/#creating-and-importing-projects),\nup to the creation of the variable `KUBE_INGRESS_BASE_DOMAIN`.\n\n2. Or, just create an ingress in your Kubernetes cluster by following the\ninstructions in our [Auto DevOps with GKE\ndocumentation](https://docs.gitlab.com/ee/topics/autodevops/cloud_deployments/auto_devops_with_gke.html#install-ingress).\n\n\n**NOTE:** For this article, we used the first method above to install an\ningress and cert-manager in the Kubernetes cluster.\n\n\n## Creating necessary project files using GitLab Duo Chat\n\n\nWe started our endeavor from VS Code and an empty project called\n`quarkus-native`, which we had previously created in GitLab and had already\ncloned to our local laptop.\n\n\n1. We opened GitLab Duo Chat, within VS Code, and entered the following\nprompt:\n\n\n**_Create a “Hello World” Quarkus application that can be natively\ncompiled_**\n\n\n![hello-world-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/hello-world-prompt.png)\n\n\nChat replied with the prerequisites and process of what to do to create a\nsimple “Hello World” Quarkus application. At this point, we were after the\ngenerated body of the class `HelloResource.java`:\n\n\n```\n\npackage org.acme;\n\n\nimport javax.ws.rs.GET;\n\nimport javax.ws.rs.Path;\n\nimport javax.ws.rs.Produces;\n\nimport javax.ws.rs.core.MediaType;\n\n\n@Path(\"/hello\")\n\npublic class HelloResource {\n\n    @GET\n    @Produces(MediaType.TEXT_PLAIN)\n    public String hello() {\n        return \"Hello World\";\n    }\n}\n\n```\n\n\n2. We knew that for Quarkus we needed a special Dockerfile, named\n`Dockerfile.native`, as part of the creation of the image for the\napplication executable. We then entered the following prompt in Chat:\n\n\n**_Create the Dockerfile.native file for the sample application above_**\n\n\n![create-dockerfile-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675941/Blog/Content%20Images/create-dockerfile-prompt.png)\n\n\nChat replied with a very thorough answer about this file, its contents, and\nthe containerization process. At this point, we were just after the\ngenerated body of the file `Dockerfile.native`:\n\n\n```\n\n####\n\n# This Dockerfile is used in order to build a container that runs the\nQuarkus application in native (no JVM) mode\n\n####\n\nFROM registry.access.redhat.com/ubi8/ubi-minimal:8.6\n\nWORKDIR /work/\n\nRUN chown 1001 /work \\\n    && chmod \"g+rwX\" /work \\\n    && chown 1001:root /work\nCOPY --chown=1001:root target/*-runner /work/application\n\n\nEXPOSE 8080\n\nUSER 1001\n\n\nCMD [\"./application\", \"-Dquarkus.http.host=0.0.0.0\"]\n\n```\n\n\n3. Next, we asked Chat to generate the `pom.xml` for this application by\nentering the following prompt:\n\n\n**_Create the pom.xml file for the sample application above_**\n\n\n![create-pom-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/create-pom-prompt.png)\n\n\nChat provided the content of the `pom.xml` file, an explanation of its\ncontents, and how to use it. At this point, we were after the generated body\nof the file `pom.xml`:\n\n\n```\n\n\u003C?xml version=\"1.0\"?>\n\n\u003Cproject xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0\nhttps://maven.apache.org/xsd/maven-4.0.0.xsd\"\nxmlns=\"http://maven.apache.org/POM/4.0.0\"\n    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n  \u003CmodelVersion>4.0.0\u003C/modelVersion>\n  \u003CgroupId>org.acme\u003C/groupId>\n  \u003CartifactId>hello-world-quarkus\u003C/artifactId>\n  \u003Cversion>1.0.0-SNAPSHOT\u003C/version>\n  \u003Cproperties>\n    \u003Ccompiler-plugin.version>3.10.1\u003C/compiler-plugin.version>\n    \u003Cmaven.compiler.release>11\u003C/maven.compiler.release>\n    \u003Cproject.build.sourceEncoding>UTF-8\u003C/project.build.sourceEncoding>\n    \u003Cproject.reporting.outputEncoding>UTF-8\u003C/project.reporting.outputEncoding>\n    \u003Cquarkus.platform.artifact-id>quarkus-bom\u003C/quarkus.platform.artifact-id>\n    \u003Cquarkus.platform.group-id>io.quarkus.platform\u003C/quarkus.platform.group-id>\n    \u003Cquarkus.platform.version>2.16.5.Final\u003C/quarkus.platform.version>\n    \u003CskipITs>true\u003C/skipITs>\n    \u003Csurefire-plugin.version>3.0.0-M7\u003C/surefire-plugin.version>\n  \u003C/properties>\n  \u003CdependencyManagement>\n    \u003Cdependencies>\n      \u003Cdependency>\n        \u003CgroupId>${quarkus.platform.group-id}\u003C/groupId>\n        \u003CartifactId>${quarkus.platform.artifact-id}\u003C/artifactId>\n        \u003Cversion>${quarkus.platform.version}\u003C/version>\n        \u003Ctype>pom\u003C/type>\n        \u003Cscope>import\u003C/scope>\n      \u003C/dependency>\n    \u003C/dependencies>\n  \u003C/dependencyManagement>\n  \u003Cdependencies>\n    \u003Cdependency>\n      \u003CgroupId>io.quarkus\u003C/groupId>\n      \u003CartifactId>quarkus-arc\u003C/artifactId>\n    \u003C/dependency>\n    \u003Cdependency>\n      \u003CgroupId>io.quarkus\u003C/groupId>\n      \u003CartifactId>quarkus-resteasy\u003C/artifactId>\n    \u003C/dependency>\n    \u003Cdependency>\n      \u003CgroupId>io.quarkus\u003C/groupId>\n      \u003CartifactId>quarkus-junit5\u003C/artifactId>\n      \u003Cscope>test\u003C/scope>\n    \u003C/dependency>\n    \u003Cdependency>\n      \u003CgroupId>io.rest-assured\u003C/groupId>\n      \u003CartifactId>rest-assured\u003C/artifactId>\n      \u003Cscope>test\u003C/scope>\n    \u003C/dependency>\n  \u003C/dependencies>\n  \u003Cbuild>\n    \u003Cplugins>\n      \u003Cplugin>\n        \u003CgroupId>${quarkus.platform.group-id}\u003C/groupId>\n        \u003CartifactId>quarkus-maven-plugin\u003C/artifactId>\n        \u003Cversion>${quarkus.platform.version}\u003C/version>\n        \u003Cextensions>true\u003C/extensions>\n        \u003Cexecutions>\n          \u003Cexecution>\n            \u003Cgoals>\n              \u003Cgoal>build\u003C/goal>\n              \u003Cgoal>generate-code\u003C/goal>\n              \u003Cgoal>generate-code-tests\u003C/goal>\n            \u003C/goals>\n          \u003C/execution>\n        \u003C/executions>\n      \u003C/plugin>\n      \u003Cplugin>\n        \u003CartifactId>maven-compiler-plugin\u003C/artifactId>\n        \u003Cversion>${compiler-plugin.version}\u003C/version>\n        \u003Cconfiguration>\n          \u003CcompilerArgs>\n            \u003Carg>-parameters\u003C/arg>\n          \u003C/compilerArgs>\n        \u003C/configuration>\n      \u003C/plugin>\n      \u003Cplugin>\n        \u003CartifactId>maven-surefire-plugin\u003C/artifactId>\n        \u003Cversion>${surefire-plugin.version}\u003C/version>\n        \u003Cconfiguration>\n          \u003CsystemPropertyVariables>\n            \u003Cjava.util.logging.manager>org.jboss.logmanager.LogManager\u003C/java.util.logging.manager>\n            \u003Cmaven.home>${maven.home}\u003C/maven.home>\n          \u003C/systemPropertyVariables>\n        \u003C/configuration>\n      \u003C/plugin>\n      \u003Cplugin>\n        \u003CartifactId>maven-failsafe-plugin\u003C/artifactId>\n        \u003Cversion>${surefire-plugin.version}\u003C/version>\n        \u003Cexecutions>\n          \u003Cexecution>\n            \u003Cgoals>\n              \u003Cgoal>integration-test\u003C/goal>\n              \u003Cgoal>verify\u003C/goal>\n            \u003C/goals>\n            \u003Cconfiguration>\n              \u003CsystemPropertyVariables>\n                \u003Cnative.image.path>${project.build.directory}/${project.build.finalName}-runner\u003C/native.image.path>\n                \u003Cjava.util.logging.manager>org.jboss.logmanager.LogManager\u003C/java.util.logging.manager>\n                \u003Cmaven.home>${maven.home}\u003C/maven.home>\n              \u003C/systemPropertyVariables>\n            \u003C/configuration>\n          \u003C/execution>\n        \u003C/executions>\n      \u003C/plugin>\n    \u003C/plugins>\n  \u003C/build>\n  \u003Cprofiles>\n    \u003Cprofile>\n      \u003Cid>native\u003C/id>\n      \u003Cactivation>\n        \u003Cproperty>\n          \u003Cname>native\u003C/name>\n        \u003C/property>\n      \u003C/activation>\n      \u003Cproperties>\n        \u003CskipITs>false\u003C/skipITs>\n        \u003Cquarkus.package.type>native\u003C/quarkus.package.type>\n      \u003C/properties>\n    \u003C/profile>\n  \u003C/profiles>\n\u003C/project>\n\n```\n\n\n4. We then asked Chat to generate the Quarkus test file for this application\nby entering the following prompt:\n\n\n**_Create the content for the Quarkus test for the application above_**\n\n\n![create-quarkus-test-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/create-quarkus-test-prompt.png)\n\n\nChat returned the body for the Quarkus test, how to use it, and the body of\nthe Java class for the test needed for its native compilation. At this\npoint, we were after the generated bodies of the file\n`HelloResourceTest.java` and `HelloResourceIT.java`.\n\n\nHere is the generated body for `HelloResourceTest.java`:\n\n\n```\n\npackage org.acme;\n\n\nimport io.quarkus.test.junit.QuarkusTest;\n\nimport org.junit.jupiter.api.Test;\n\n\nimport static io.restassured.RestAssured.given;\n\nimport static org.hamcrest.CoreMatchers.is;\n\n\n@QuarkusTest\n\npublic class HelloResourceTest {\n\n    @Test\n    public void testHelloEndpoint() {\n        given()\n          .when().get(\"/hello\")\n          .then()\n             .statusCode(200)\n             .body(is(\"Hello World\"));\n    }\n\n}\n\n```\n\n\nHere is the generated body for `HelloResourceIT.java`:\n\n\n```\n\npackage org.acme;\n\n\nimport io.quarkus.test.junit.QuarkusIntegrationTest;\n\n\n@QuarkusIntegrationTest\n\npublic class HelloResourceIT extends HelloResourceTest {\n    // Execute the same tests but in native mode.\n}\n\n```\n\n\n5. We needed to know how to organize these files in the GitLab project, so\nwe asked about the directory structure for all these files by entering the\nfollowing prompt in Chat:\n\n\n**_Give me the entire directory structure for this project including the\nlocation of each file, e.g. pom.xml, Dockerfile.native,\napplication.properties, HelloResource.java, HelloResourceTest.java, and the\nlocation of the target directory_**\n\n\n![create-dir-struct-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/create-dir-struct-prompt.png)\n\n\nChat replied with a detailed diagram about the entire directory structure\nfor the project and where all these files should be located as well as a\ndescription of the purpose of each of them. It even mentioned that the\ndirectory `target/` and its contents should not be version controlled since\nit was generated by the build process. Another interesting aspect of the\nreply was the existence of a file called `resources/application.properties`\nin the directory structure.\n\n\n![dir-struct-chat-response](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/dir-struct-chat-response.png)\n\n\nWith all this information in our hands, we were ready to start creating\nthese files in our GitLab project.\n\n\n## Populating our project with the generated content for each file\n\n\nWe created each of the following files in their corresponding location and\ntheir generated content as provided by Chat:\n\n\n- `src/main/java/org/acme/HelloResource.java`\n\n- `resources/application.properties`\n\n- `src/test/java/org/acme/HelloResourceTest.java`\n\n- `src/test/java/org/acme/HelloResourceIT.java`\n\n- `pom.xml`\n\n- `Dockerfile.native`\n\n\n**NOTE:** We considered using GitLab Auto Deploy for this endeavor but later\nrealized that it would not be a supported option. We are mentioning this\nbecause in the video at the end of this tutorial, you will see that we asked\nChat: `How to set the service internalPort to 8080 for auto deploy`. Then we\ncreated a file named `.gitlab/auto-deploy-values.yaml` with the generated\ncontent from Chat. The creation of this file is not necessary for this\ntutorial.\n\n\nBefore we started tackling the pipeline to build, containerize, and deploy\nthe application to our Kubernetes cluster, we decided to generate the\nexecutable locally on our Mac and test the application locally.\n\n\n## Testing the application locally\n\n\nHere is the process we went through to test the application on our local\nmachine.\n\n\n1. To build the application on the local Mac laptop, from a Terminal window,\nwe entered the following command:\n\n\n```\n\nmvn clean package -Pnative\n\n```\n\n\n![first-build](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/first-build.png)\n\n\nThe native compilation failed with the error message:\n\n\n`Cannot find the ‘native-image’ in the GRAALVM_HOME, JAVA_HOME and System\nPATH. Install it using ‘gu install native-image’`\n\n\n2. So, we used our trusty GitLab Duo Chat again and asked it the following:\n\n\n**_The command “mvn clean package -Pnative” is failing with error\n“java.lang.RuntimeException: Cannot find the ‘native-image’ in the\nGRAALVM_HOME, JAVA_HOME and System PATH. Install it using gu install\nnative-image”. I’m using a MacOS Sonoma. How do I fix this error on my\nMac?_**\n\n\n![how-to-fix-build-failure-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/how-to-fix-build-failure-prompt.png)\n\n\nChat replied with a detailed set of steps on how to install the necessary\nsoftware and set the appropriate environment variables.\n\n\n3. We copied and pasted the following commands from the Chat window to a\nTerminal window:\n\n\n```\n\nbrew install –cask graalvm/tap/graalvm-ce-java17\n\nexport JAVA_HOME=/Library/Java/JavaVIrtualMachines/graalvm-ce-java17-22.3.1\n\nexport GRAALVM_HOME=${JAVA_HOME}\n\nexport PATH=${GRAALVM_HOME}/bin:$PATH\n\nxattr -r -d com.apple.quarantine ${GRAALVM_HOME}/../..\n\ngu install native-image\n\n```\n\n\nThe commands above installed the community edition of GraalVM Version 22.3.1\nthat supported Java 17. We noticed, during the brew install, that the\nversion of the GraalVM being installed was `java17-22.3.1`, so we had to\nupdate the pasted value for `JAVA_HOME` from `graalvm-ce-java17-22.3.0` to\n`graalvm-ce-java17-22.3.1`.\n\n\nWe also had to run the `xattr` command to get the GraalVM, which we had\ndownloaded and installed on our Mac, out of quarantine so that it could run\nlocally. Lastly, we installed the GraalVM native-image.\n\n\n4. At this point, we again, from a Terminal window, entered the following\ncommand to build the application on the local Mac laptop:\n\n\n```\n\nmvn clean package -Pnative\n\n```\n\n\nThis time the compilation was successful and an executable was generated in\nthe `target` directory.\n\n\n![successful-local-compilation](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/successful-local-compilation.png)\n\n\n5. We ran the executable by entering the following commands from a Terminal\nwindow:\n\n\n```\n\ncd target\n\n./quarkus-native-1.0.0-SNAPSHOT-runner “-Dquarkus.http.host=0.0.0.0”\n\n```\n\n\n![executable-local-run](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/executable-local-run.png)\n\n\n6. With the application running, we opened a browser window, and in the URL\nfield, we entered:\n\n\n```\n\nhttp://localhost:8080/hello\n\n```\n\n\n![app-running-locally](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/app-running-locally.png)\n\n\nThe application returned the string `Hello World`, which was displayed in\nthe browser window.\n\n\nAt this point, we committed and pushed all the changes to our GitLab project\nand started working on creating a CI/CD pipeline that would build and deploy\nthe application to a Kubernetes cluster running on the cloud.\n\n\nBut before continuing, we remembered to add, commit, and push a `.gitignore`\nfile to our project that included the path `target/`, since this was the\ndirectory where the executable would be created and we didn’t need to keep\nit - or its contents - under version control.\n\n\n## Creating the pipeline with GitLab Duo Chat\n\n\nNow that we had already successfully tested the application locally on our\nMac, we needed to create the CI/CD pipeline that would compile the\napplication, containerize it, and deploy it to our Kubernetes cluster. We\nwanted to keep the pipeline simple, brief, and have a single environment in\nwhich to deploy it. To this end, the pipeline would not tackle multiple\nenvironments or feature branches, for example.\n\n\n1. To avoid manually creating a pipeline from scratch, we decided to once\nagain leverage Chat. We entered the following prompt\n\n\n**_Create a .gitlab-ci.yml file with 3 stages: build, containerize, and\ndeploy. Each of these stages should have a single job with the same name.\nThe build job should compile the application natively using the -Pnative\nmaven option and the builder image for mandrel jdk-22 for java17 and store\nthe application executable and its Dockerfile as artifacts. The containerize\njob should use docker to build and push the image to the built-in container\nregistry. The deploy job should rollout the containerized application, named\nquarkus-app, to the production environment in the Kubernetes cluster by\ncreating a deployment resource, a service resource and an ingress rule with\nhost ${KUBE_INGRESS_BASE_DOMAIN} and /hello prefix. The deploy job should\nalso include the url for the production environment. When setting the\nKubernetes context, use ${KUBE_CONTEXT}, and for application namespace, use\n${KUBE_NAMESPACE}_**\n\n\n![create-pipeline-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/create-pipeline-prompt.png)\n\n\nThe prompt above provided some points that we thought would be useful and\nnecessary for a good outcome. The requirements listed by the prompt were:\n\n\n- It specified that the pipeline needed to have three stages: build,\ncontainerize, and deploy.\n\n- Each stage would have a single job.\n\n- The build job should natively compile (not build) the application using\nthe -Pnative option.\n\n- The build job should use the open source software `mandrel` and use the\nversion that would support jdk-22 for Java 17. Remember that these were the\nversions that we tested on our local machine.\n\n- The build job would also need to pass the resulting executable and the\nproject Dockerfile to the next job, i.e. containerize, in the pipeline.\n\n- The containerize job should build a docker image with the executable using\nthe Dockerfile, both of which were passed as artifacts.\n\n- The containerize job should push the image to the GitLab container\nregistry.\n\n- The deploy job should use the image from the GitLab container registry,\nname it `quarkus-app`, and deploy it to the Kubernetes cluster using\ndeployment, service and ingress rule resources.\n\n- The deploy job should use the environment variables\n`${KUBE_INGRESS_BASE_DOMAIN}`, `${KUBE_CONTEXT}`, and `${KUBE_NAMESPACE}`,\nwhich had been defined as group/project variables, accordingly.\n\n\nChat responded with the following `.gitlab-ci.yml` example:\n\n\n```\n\n# Define the stages for the pipeline\n\nstages:\n  - build\n  - containerize\n  - deploy\n\n# Variables section to use GitLab CI/CD variables\n\nvariables:\n  MAVEN_OPTS: \"-Dmaven.repo.local=.m2/repository\"\n  MAVEN_CLI_OPTS: \"--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true\"\n  CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}\n\n# Cache configuration for Maven dependencies\n\ncache:\n  paths:\n    - .m2/repository\n\n# Build stage: Compile the application natively\n\nbuild:\n  stage: build\n  image:\n    name: quay.io/quarkus/ubi-quarkus-mandrel-builder-image:22.3-java17\n    entrypoint:\n      - \"\"\n  script:\n    - ./mvnw ${MAVEN_CLI_OPTS} clean package -Pnative\n  artifacts:\n    paths:\n      - target/*-runner\n      - src/main/docker/Dockerfile.native-micro\n  # This job runs only on the main branch\n  only:\n      - main\n\n# Containerize stage: Build and push Docker image\n\ncontainerize:\n  stage: containerize\n  image: docker:20.10.16\n  services:\n    - docker:20.10.16-dind\n  variables:\n    DOCKER_TLS_CERTDIR: \"/certs\"\n  before_script:\n    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY\n  script:\n    - docker build -f src/main/docker/Dockerfile.native-micro -t ${CONTAINER_IMAGE} .\n    - docker push ${CONTAINER_IMAGE}\n  # This job runs only on the main branch\n  only:\n      - main\n\n# Deploy stage: Deploy to Kubernetes\n\ndeploy:\n  stage: deploy\n  image: \n    name: bitnami/kubectl:latest\n    entrypoint: ['']\n  script:\n    - kubectl config use-context ${KUBE_CONTEXT}\n    - kubectl config set-context --current --namespace=${KUBE_NAMESPACE}\n    - |\n      cat \u003C\u003CEOF | kubectl apply -f -\n      apiVersion: apps/v1\n      kind: Deployment\n      metadata:\n        name: quarkus-app\n      spec:\n        replicas: 1\n        selector:\n          matchLabels:\n            app: quarkus-app\n        template:\n          metadata:\n            labels:\n              app: quarkus-app\n          spec:\n            containers:\n            - name: quarkus-app\n              image: ${CONTAINER_IMAGE}\n              ports:\n              - containerPort: 8080\n      ---\n      apiVersion: v1\n      kind: Service\n      metadata:\n        name: quarkus-app\n      spec:\n        selector:\n          app: quarkus-app\n        ports:\n        - port: 80\n          targetPort: 8080\n      ---\n      apiVersion: networking.k8s.io/v1\n      kind: Ingress\n      metadata:\n        name: quarkus-app\n      spec:\n        rules:\n        - host: ${KUBE_INGRESS_BASE_DOMAIN}\n          http:\n            paths:\n            - path: /hello\n              pathType: Prefix\n              backend:\n                service:\n                  name: quarkus-app\n                  port: \n                    number: 80\n      EOF\n    - echo \"Application deployed to https://${KUBE_INGRESS_BASE_DOMAIN}/hello\"\n  # This job runs only on the main branch\n  only:\n      - main\n```\n\n\n2. There were some things we needed to adjust in the sample `.gitlab-ci.yml`\nfile above before we could commit it to our `main` branch. These are the\nupdates we made to the file:\n\n\n- We deleted all occurrences of `only: -main` because we wanted to keep of\npipeline definition file simple and with no branch-related rules.\n\n- We fixed the name of the file `Dockerfile.native-micro` to\n`Dockerfile.native`.\n\n\n3. At this point, we wanted to ensure that the deployment would be to the\n`production` environment so we asked Chat the following prompt:\n\n\n**_What is the syntax to specify an environment with its url in a\npipeline?_**\n\n\n![how-to-add-env-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/how-to-add-env-prompt.png)\n\n\nThe response from Chat included an example of how to do this so we used this\ninformation to add the following environment block to our pipeline:\n\n\n```\n  environment:\n       name: production\n       url: http://${KUBE_INGRESS_BASE_DOMAIN}/hello\n```\n\n\n4. The example provided by Chat includes a URL that started with `https` and\nwe modified that to `http` since we didn’t really need a secure connection\nfor this simple application.\n\n\n5. Lastly, we noticed that in the `build` job, there was a script `mvnw`\nthat we didn’t have in our project. So, we asked Chat the following:\n\n\n**_How can I get the mvnw script for Quarkus?_**\n\n\n![how-to-add-mvnw-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/how-to-add-mvnw-prompt.png)\n\n\nChat responded with the command to execute to bootstrap and create this\nscript. We executed this command from a Terminal window:\n\n\n```\n\nmvn wrapper:wrapper\n\n```\n\n\nWe were now ready to commit all of our changes to the `main` branch and have\nthe pipeline executed. However, on our first attempt, our first pipeline\nfailed at the build job.\n\n\n## Troubleshooting using GitLab Duo Root Cause Analysis\n\n\nOur first attempt at running our brand-new pipeline failed. So, we took\nadvantage of [GitLab Duo Root Cause\nAnalysis](https://about.gitlab.com/blog/developing-gitlab-duo-blending-ai-and-root-cause-analysis-to-fix-ci-cd/),\nwhich looks at the job logs and provides a thorough natural language\nexplanation (with examples) of the root cause of the problem and, most\nimportantly, how to fix it.\n\n\n![build-job-troubleshooting](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/build-job-troubleshooting.png)\n\n\nRoot Cause Analysis recommended we look at the compatibility of the command\nthat was trying to be executed with the image of mandrel used in the build\njob. We were not using any command with the image so we concluded that it\nmust have been the predefined `entrypoint` for the image itself. We needed\nto override this so we asked Chat the following:\n\n\n**_How do I override the entrypoint of an image using gitlab keywords?_**\n\n\n![how-to-override-entrypoint-prompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/how-to-override-entrypoint-prompt.png)\n\n\nChat replied with some use case examples of overriding an image entry point.\nWe used that information to update the build job image definition:\n\n\n```\n\nbuild:\n    stage: build\n    image: quay.io/quarkus/ubi-quarkus-mandrel-builder-image:22.3-java17\n    entrypoint:\n        - “”\n```\n\n\nWe committed our changes to the `main` branch, which launched a new instance\nof the pipeline. This time the build job executed successfully but the\npipeline failed at the `containerize` job.\n\n\n## Running a successful pipeline\n\n\nBefore drilling down into the log of the failed `containerize` job, we\ndecided to drill into the log of the successfully completed build job first.\nEverything looked good in the log of the build job with the exception of\nthis warning message at the very end of it:\n\n\n```\n\nWARNING: src/main/docker/Dockerfile.native: no matching files. Ensure that\nthe artifact path is relative to the working directory …\n\n``` \n\n\nWe took notice of this warning and then headed to the log of the failed\n`containerize` job. In it, we saw that the `docker build` command had failed\ndue to a non-existent Dockerfile. We ran Root Cause Analysis on the job and\namong its suggested fixes was for us to verify that the project structure\nmatched the path of the specified `Dockerfile.native` file.\n\n\n![containerize-job-troubleshooting](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/containerize-job-troubleshooting.png)\n\n\nThis information confirmed our suspicion of the misplaced\n`Dockerfile.native` file. Instead of being at the directory\n`src/main/docker` as specified in the pipeline, it was located at the root\ndirectory of the project.\n\n\nSo, we went back to our project and updated every occurrence of the location\nof this file in our `.gitlab-ci.yml` file. We modified the two locations\nwhere this happened, one in the `build` job and one in the `containerize`\njob, as follows:\n\n\n```\n\nsrc/main/docker/Dockerfile.native\n\n```\n\n\nto\n\n\n```\n\nDockerfile.native\n\n```\n\n\nWe committed our updates to the `main` branch and this time our entire\npipeline executed successfully!\n\n\n![pipeline-successful-run](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/pipeline-successful-run.png)\n\n\nOur last step was to check the running application in the `production`\nenvironment in our Kubernetes cluster.\n\n\n## Accessing the deployed application running in cluster\n\n\nOnce the pipeline ran successfully to completion, we drilled in the log file\nfor the `deploy` job. Remember, this job printed the URL of the application\nat the end of its execution. We scrolled down to the bottom of the log and\nclicked on the `https` application link, which opened a browser window\nwarning us that the connection was not private (we disabled `https` for the\nenvironment URL but forgot it for this string). We proceeded past the\nbrowser warning and then the string \"Hello World\" was displaced in the\nbrowser window indicating that the application was up and running in the\nKubernetes cluster.\n\n\nFinally, to double-check our production deployment URL, we headed to the\nproject **Operate > Environments** window, and clicked on the \"Open\" button\nfor it, which immediately opened a browser window with the \"Hello World\"\nmessage.\n\n\n![app-running-on-k8s](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749675940/Blog/Content%20Images/app-running-on-k8s.png)\n\n\n## Try it \n\n\nWe created, compiled, built, and deployed a simple Quarkus application to a\nKubernetes cluster using [GitLab Duo](https://about.gitlab.com/gitlab-duo/).\nThis approach allowed us to be more efficient and productive in all the\ntasks that we performed and it helped us streamline our DevSecOps processes.\nWe have shown only a small portion of how GitLab Duo's AI-powered\ncapabilities can help you, namely Chat and Root Cause Analysis. There’s so\nmuch more you can leverage in GitLab Duo to help you create better software\nfaster and more securely.\n\n\nWatch this whole use case in action:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/xDpycxz3RPY?si=HHZrFt1O_8XoLATf\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nAll the project assets we used are available\n[here](https://gitlab.com/gitlab-da/use-cases/ai/ai-applications/quarkusn/quarkus-native).\n\n\n> [Try GitLab Duo for free](https://about.gitlab.com/solutions/gitlab-duo-pro/sales/?type=free-trial&toggle=gitlab-duo-pro)\nand get started on exciting projects like this.\n",[720,9,851,1608,696,694,108],{"slug":4532,"featured":90,"template":700},"use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project","content:en-us:blog:use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project.yml","Use Gitlab Duo To Build And Deploy A Simple Quarkus Native Project","en-us/blog/use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project.yml","en-us/blog/use-gitlab-duo-to-build-and-deploy-a-simple-quarkus-native-project",{"_path":4538,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4539,"content":4545,"config":4550,"_id":4552,"_type":13,"title":4553,"_source":15,"_file":4554,"_stem":4555,"_extension":18},"/en-us/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance",{"title":4540,"description":4541,"ogTitle":4540,"ogDescription":4541,"noIndex":6,"ogImage":4542,"ogUrl":4543,"ogSiteName":685,"ogType":686,"canonicalUrls":4543,"schema":4544},"Use GitLab Duo Workflow to improve application quality assurance","Learn step-by-step how to add unit tests to a Java application using agentic AI (includes a video tutorial).","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097617/Blog/Hero%20Images/Blog/Hero%20Images/Workflow%201800x945_2gQoQIbY9NvjLFpXtsxtXy_1750097616649.png","https://about.gitlab.com/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Use GitLab Duo Workflow to improve application quality assurance\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Cesar Saavedra\"}],\n        \"datePublished\": \"2025-04-10\",\n      }",{"title":4540,"description":4541,"authors":4546,"heroImage":4542,"date":4547,"body":4548,"category":718,"tags":4549},[715],"2025-04-10","Assuring the quality of your applications via test-driven design, good code\ncoverage, and issue detection is critically important to your customers and\nyour reputation, but it can also be a time-consuming endeavor. [GitLab Duo\nWorkflow](https://about.gitlab.com/gitlab-duo/agent-platform/), agentic AI built\non top of the most comprehensive DevSecOps platform, can help you quickly\ncomplete development tasks such as adding unit tests to a Java application.\nThis tutorial demonstrates how by using this sample [Java\nproject](https://gitlab.com/gitlab-da/playground/csaavedra/gdw/prodmgr-gdw).\n\n\n> GitLab Duo Workflow is currently in private beta. Join the\n[waitlist](https://about.gitlab.com/gitlab-duo/agent-platform/) to see what’s\npossible with AI agents that understand your entire SDLC.\n\n\n## Opening your project in VS Code\n\n\n1. Open the Java project in Visual Studio Code (after cloning it to your\nlocal machine). Ensure that you’re in a feature branch (not the main or\ndefault branch) before you start. If you’re already working on a merge\nrequest, it will have its own associated feature branch.\n\n\n2. (This step is optional.) Navigate to the file that defines the Java class\nfor which you’d like to have GitLab Duo Workflow create unit tests. Inspect\nit so that you can later confirm that the generated unit tests do cover its\nclass members. This is what you would see:\n\n\n![File that defines the Java class for which you’d like to have GitLab Duo\nWorkflow create unit\ntests](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097627/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097627482.png)\n\n\n**Note:** We are assuming that you already enabled the GitLab Duo Workflow\nextension in your VS Code. If not, please refer to the [setup\ndocumentation](https://docs.gitlab.com/user/duo_workflow/#use-workflow-in-vs-code).\n\n\n3. Launch GitLab Duo Workflow by opening the VS Code command palette [Ctrl +\nShift + P] and entering \"GitLab Duo Workflow\" in it and selecting **GitLab:\nShow Duo Workflow**. A tab will appear that looks like this:\n\n\n![Launching GitLab Duo Workflow with VS\nCode](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097628/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097627483.png)\n\n\n4. The next step is to add tests for the default constructor, the\nverification of the object creation, and the initial state of the properties\nof the Product class. To accomplish this, enter the following prompt in the\ntext area in GitLab Duo Workflow:\n\n\n```unset\n\nCreate unit tests for class defined in the Product.java file and store the\nunit tests in its own file titled ProductTest.java\n\n```\n\n\n![Prompt area in GitLab Duo\nWorkflow](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097628/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097627484.png)\n\n\n5. Click the **Start** button in the GitLab Duo Workflow window. Two new\nwindows will appear: one in the center of the screen and one to the right.\nThe one on the right displays the analysis that GitLab Duo Workflow is\nperforming to come up with a plan that will achieve the goal as specified in\nyour prompt. The plan is displayed in the center window. After the analysis\nand the plan are finished, you should see an output like this:\n\n\n![Analysis and plan generated by GitLab Duo\nWorkflow](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097627/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097627486.png)\n\n\n6. Review the analysis and plan and, if you are satisfied with them, click\n**Approve plan** at the bottom of the window.\n\n\n7. GitLab Duo Workflow will start executing the approved plan and making\nmodifications to your project accordingly.\n\n\n8. Once the execution of the plan is finished, you will see a new directory\n`src/test/java/csaa/jspring/ProductManager` in the project with a new file\nin it named `ProductTest.java`, which contains all the unit tests for the\n`Product.java` class.\n\n\n![New directory in the project iwth a new file name\n`ProductTest.java`](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097628/Blog/Content%20Images/Blog/Content%20Images/image7_aHR0cHM6_1750097627488.png)\n\n\n9. Navigate to the newly created file `ProductTest.java` and you will see\nthat it has some import statements underlined in red indicating some import\nerrors:\n\n\n![`ProductTest.java` include imports statement and error indicators in\nred](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097628/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097627489.png)\n\n\nLet’s have GitLab Duo Workflow fix these for us.\n\n\n**Note:** We could have also asked GitLab Duo Workflow in our first prompt\nto update the `pom.xml` file accordingly. But since we didn’t, let’s fix\nthese errors in a new workflow.\n\n\n## Launching a GitLab Duo Workflow to fix errors in generated code\n\n\n10. Start a new workflow by clicking on the **New workflow** button at the\nbottom of the analysis window on the right side of your screen.\n\n\n![New workflow\nbutton](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097628/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097627491.png)\n\n\n11. In the prompt text area, enter the following:\n\n\n```unset\n\nThe file ProductTest.java has an error “The import org.junit cannot be\nresolved”. Please fix it\n\n```\n\n\n12. After you approve the proposed plan, GitLab Duo Workflow starts its\nanalysis by reading the current `pom.xml` file. It then edits it and removes\nthe outdated JUnit dependency, and follows that with the addition of the\ncorrect dependency and version for JUnit. Lastly, it reads the\n`ProductTest.java` file to clear all the dependency errors.\n\n\n![GitLab Duo Workflow carrying out analysis by reading\npom.xml](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097627/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097627492.png)\n\n\n## Watch the tutorial\n\n\nThrough the execution of this plan, GitLab Duo Workflow is effectively\nmaking updates to the project to achieve what was requested in the prompt,\nsaving time and effort, and increasing productivity so that developers can\nspend more time innovating and creating value for their organization.\n\n\nIf you’d like to see what you read above in action, watch the following\nvideo:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/Tuj7TgqY81Q?si=RReuL1pUsLafvAzs\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n> Sign up for the [GitLab Duo Workflow private beta\nwaitlist](https://about.gitlab.com/gitlab-duo/agent-platform/) to see what’s\npossible with AI agents that understand your entire SDLC.\n\n\n## Read more about GitLab Duo Workflow and agentic AI\n\n\n- [GitLab Duo Workflow: Enterprise visibility and control for agentic\nAI](https://about.gitlab.com/blog/gitlab-duo-workflow-enterprise-visibility-and-control-for-agentic-ai/)\n\n- [GitLab Duo Workflow\ndocumentation](https://docs.gitlab.com/user/duo_workflow/)\n\n- [GitLab Duo](https://about.gitlab.com/gitlab-duo/)\n\n- [Agentic AI: Unlocking developer potential at scale (The\nSource)](https://about.gitlab.com/the-source/ai/agentic-ai-unlocking-developer-potential-at-scale/)\n",[720,495,9,694,696],{"slug":4551,"featured":6,"template":700},"use-gitlab-duo-workflow-to-improve-application-quality-assurance","content:en-us:blog:use-gitlab-duo-workflow-to-improve-application-quality-assurance.yml","Use Gitlab Duo Workflow To Improve Application Quality Assurance","en-us/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance.yml","en-us/blog/use-gitlab-duo-workflow-to-improve-application-quality-assurance",{"_path":4557,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4558,"content":4564,"config":4568,"_id":4570,"_type":13,"title":4571,"_source":15,"_file":4572,"_stem":4573,"_extension":18},"/en-us/blog/use-gitlab-to-detect-vulnerabilities",{"title":4559,"description":4560,"ogTitle":4559,"ogDescription":4560,"noIndex":6,"ogImage":4561,"ogUrl":4562,"ogSiteName":685,"ogType":686,"canonicalUrls":4562,"schema":4563},"How to use GitLab security features to detect log4j vulnerabilities","Detailed guidance to help customers detect vulnerabilities.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749666816/Blog/Hero%20Images/security-cover.png","https://about.gitlab.com/blog/use-gitlab-to-detect-vulnerabilities","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use GitLab security features to detect log4j vulnerabilities\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"GitLab\"}],\n        \"datePublished\": \"2021-12-15\",\n      }",{"title":4559,"description":4560,"authors":4565,"heroImage":4561,"date":1687,"body":4566,"category":300,"tags":4567},[868],"_Note: Out of an abundance of caution, we encourage users who are using\nolder versions of GitLab SAST and Dependency Scanning to update to the\nlatest versions. You can find more information and recommended actions in\n[this blog\npost](/blog/updates-and-actions-to-address-logj-in-gitlab/)._\n\n\n_Any customer leveraging the [recommended\nincludes](https://docs.gitlab.com/ee/user/application_security/sast/#configure-sast-in-your-cicd-yaml)\nfor GitLab SAST has automatically received the new patched versions released\nDec 13, 2021._\n\n\nIn light of the recently discovered log4j vulnerabilities, we would like to\ndemonstrate how GitLab can be used to assess and remediate the log4j\nvulnerability as well as other security vulnerabilities that may exist in\nyour projects.\n\n\nThe solutions shared here are: \n\n* [Dependency Scanning\n(Ultimate)](#use-gitlab-dependency-scanning-to-detect-and-mitigate-log4j-vulnerabilities)\n\n* [Container Scanning\n(Ultimate)](#detect-log4j-vulnerabilities-with-container-scanning)\n\n* [Cluster image scanning\n(Ultimate)](#detect-vulnerable-containers-in-your-kubernetes-cluster)\n\n* [Advanced Search\n(Premium)](#search-gitlab-projects-which-use-the-log4j-java-library)\n\n\nFree users wishing to access Premium and Ultimate features can do so by\nsigning up for a [free trial](https://about.gitlab.com/free-trial/) of\nGitLab. \n\n\n### Use GitLab dependency scanning to detect and mitigate log4j\nvulnerabilities \n\n\n[Dependency\nscanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning)\nuses Gemnasium, which has been\n[updated](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/merge_requests/11381)\nto detect the log4j vulnerability, to automatically find security\nvulnerabilities in your software dependencies.\n\n\nLet’s try dependency scanning with a vulnerable project. Navigate to `Create\nnew project > Import project > from URL` and use\n`https://github.com/christophetd/log4shell-vulnerable-app.git`. \n\n\nNext, navigate to `Security & Compliance > Security dashboard` and select to\nconfigure `Dependency Scanning`. This will create a new merge request\nenabling the dependency scanner, and you can immediately see the first\n[scanning\nresults](https://gitlab.com/gitlab-de/playground/log4shell-vulnerable-app/-/pipelines/427550530/security)\nin the [merge\nrequest](https://gitlab.com/gitlab-de/playground/log4shell-vulnerable-app/-/merge_requests/1). \n\n\nAlternatively, you can edit the `.gitlab-ci.yml` configuration file and\ninclude the Dependency Scanning CI/CD template.\n\n\n```yaml\n\ninclude:\n\n- template: Security/Dependency-Scanning.gitlab-ci.yml\n\n```\n\n\nCreate a new merge request and wait for the pipeline to finish. Inspect the\nsecurity reports. \n\n\n![GitLab security\nreport](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image2.png){:\n.shadow}\n\n\nTake action on the critical vulnerability, open the details and create a new\nconfidential security issue to follow-up. \n\n\n![Details of security\nvulnerability](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image9.png){:\n.shadow}\n\n\nAfter merging the MR to add dependency scanning, future MRs and code changes\nwill detect the log4j vulnerabilities. This helps to avoid accidentally\nintroducing older versions again. Open the `Security report` in `Security &\nCompliance` to get an overview of the vulnerabilities. \n\n\n![Panel showing security\nvulnerabilities](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image4.png){:\n.shadow}\n\n\nYou can customize the default settings using [CI/CD\nvariables](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings),\nfor example increasing the log level to debug with `SECURE_LOG_LEVEL:\n‘debug’`. \n\n\nThe project created in the examples above is located\n[here](https://gitlab.com/gitlab-de/playground/log4shell-vulnerable-app). \n\n\n### Detect log4j vulnerabilities with Container Scanning\n\n\nVulnerabilities in container images can come not only from the source code\nfor the application, but also from packages and libraries that are installed\non the base image. Images can inherit packages and vulnerabilities from\nother container images using the `FROM` keyword in a `Dockerfile`.\n[Container\nScanning](https://docs.gitlab.com/ee/user/application_security/container_scanning/)\nhelps detect these vulnerabilities for the Operating System including\npackages. The latest release adds language vulnerability scans as a new\noptional feature to help detect the log4j library vulnerability using the\nunderlying scanners (Trivy as default, Grype optional). You can also use\nthis capability to scan remote images using the `DOCKER_IMAGE` variable.\n\n\nYou can enable the `CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN` variable to\n[scan for language specific\npackages](https://docs.gitlab.com/ee/user/application_security/container_scanning/#report-language-specific-findings).\nPlease note that the additionally detected language dependencies can cause\nduplicates when you enable Dependency Scanning too. \n\n\nTo try it, navigate to `CI/CD > Pipeline Editor` and add the following\nconfiguration for Container Scanning:\n\n\n```yaml\n\ninclude:\n    - template: Security/Container-Scanning.gitlab-ci.yml\n\nvariables:\n    # Use Trivy or Grype as security scanners (Trivy is the default in the included template)\n    # CS_ANALYZER_IMAGE: \"registry.gitlab.com/security-products/container-scanning/trivy:4\"\n    # CS_ANALYZER_IMAGE: \"registry.gitlab.com/security-products/container-scanning/grype:4\"\n    # Detect language libraries as dependencies\n    CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN: \"false\"\n    # Test the vulnerable log4j image \n    DOCKER_IMAGE: registry.gitlab.com/gitlab-de/playground/log4shell-vulnerable-app:latest \n```\n\n\nCreate a new branch, commit the changes and create a new MR. Once the\npipeline has completed, inspect the security report in the MR. \n\n\n![List of vulnerabilities detected by container\nscanning](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image6.png){:\n.shadow}\n\n\nAfter merging the MR, you can view the vulnerabilities that exist in your\ndefault branch by navigating to `Security & Compliance > Vulnerability\nReport`. \n\n\n![Panel showing security\nvulnerabilities](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image7.png){:\n.shadow}\n\n\nInspect the vulnerability details to take action.\n\n\n![Detail on\nvulnerability](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image8.png){:\n.shadow}\n\n\nThis feature is available for customers using the default CI/CD templates,\nor the tagged `:4` scanner images from  GitLab's Container Registry\n(registry.gitlab.com). If you are using custom images, please rebuild them\nbased on the latest release.\n\n\n### Detect vulnerable containers in your Kubernetes cluster\n\n\nYou can use [cluster image scanning in\nKubernetes](https://docs.gitlab.com/ee/user/clusters/agent/vulnerabilities.html)\nwhich uses Starboard and [uses Trivy as a security\nscanner](https://aquasecurity.github.io/starboard/v0.13.1/integrations/vulnerability-scanners/trivy/)\nunder the hood. Trivy’s vulnerability DB is able to detect CVE-2021-44228.\n\n\nLet’s try it! A quick way to bring up a Kubernetes cluster is in Civo Cloud.\nCreate an account, and follow the documentation on [how to set up the\nCLI](https://www.civo.com/learn/kubernetes-cluster-administration-using-civo-cli)\nwith an API token. Next, create a k3s cluster. \n\n\n```shell\n\n$ civo kubernetes create log4j\n\n$ civo kubernetes config log4j --save\n\n$ kubectl config use-context log4j\n\n$ kubectl get node\n\n```\n\n\n`registry.gitlab.com/gitlab-de/playground/log4shell-vulnerable-app:latest`\nprovides a vulnerable container image we can deploy and then scan. \n\n\n```shell\n\n$ vim deployment.yaml\n\n\napiVersion: apps/v1\n\nkind: Deployment\n\nmetadata:\n  name: log4j\nspec:\n  replicas: 2\n  selector:\n    matchLabels:\n      app: log4j\n  template:\n    metadata:\n      labels:\n        app: log4j\n    spec:\n      containers:\n        - image: registry.gitlab.com/gitlab-de/playground/log4shell-vulnerable-app:latest\n          name: log4j\n\n$ kubectl apply -f deployment.yaml\n\n```\n\n\n```shell\n\n$ vim service.yaml\n\n\napiVersion: v1\n\nkind: Service\n\nmetadata:\n  name: log4j\n  labels:\n    app: log4j\nspec:\n  ports:\n    - name: \"log4j\"\n      port: 8080\n  selector:\n    app: log4j\n\n$ kubectl apply -f service.yaml\n\n```\n\n\nTest the application container with port forwarding, and open your browser\nat http://localhost:80808. You can close the connection with `ctrl+c`. \n\n\n```\n\n$ kubectl port-forward service/log4j 8080:8080\n\n```\n\n\nAfter the deployment is finished, let’s add the cluster image scanning\nintegration. Follow the [Starboard\nOperator](https://aquasecurity.github.io/starboard/v0.13.1/operator/installation/kubectl/)\ninstallation documentation. Next, configure the [Kubernetes Cluster Image\nScanning](https://docs.gitlab.com/ee/user/clusters/agent/vulnerabilities.html)\nwith GitLab. \n\n\nThe final step is to integrate the CI/CD template and run the pipelines. \n\n\n```yaml\n\ninclude:\n  - template: Security/Cluster-Image-Scanning.gitlab-ci.yml\n```\n\n\nNavigate into `Security & Compliance > Vulnerability report` and select the\n`Operational vulnerabilities` tab to inspect the vulnerabilities. There you\ncan see that `log4j` was detected in the deployed application running in our\nKubernetes cluster 💜. \n\n\n![Panel showing security\nvulnerabilities](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image5.png){:\n.shadow}\n\n\nInspect the `log4j` vulnerability to see more details. \n\n\n![Detail on\nvulnerability](https://about.gitlab.com/images/blogimages/2021-12-15-use-gitlab-to-detect-log4j/image3.png){:\n.shadow}\n\n\nThe full project is located\n[here](https://gitlab.com/gitlab-de/playground/log4j-kubernetes-container-scanning).\n\n\n### Search GitLab projects which use the log4j Java library\n\n\nYou can use the [advanced search with scope\nblobs](https://docs.gitlab.com/ee/api/search.html#scope-blobs). Let’s try\nit! Navigate to your profile and add a new personal access token (PAT).\nExport it into the environment to access it in the next step:\n\n\n```shell\n\n$ export GITLAB_TOKEN=xxxxxxxxx\n\n\n$ curl --header \"PRIVATE-TOKEN: $GITLAB_TOKEN\"\n\"https://gitlab.com/api/v4/search?scope=blobs&search=log4j\" \n\n```\n\n\nTip: Install jq to format the JSON body. More insights in [this blog\npost](/blog/devops-workflows-json-format-jq-ci-cd-lint/). \n\n\n```shell\n\n$ curl --header \"PRIVATE-TOKEN: $GITLAB_TOKEN\"\n\"https://gitlab.com/api/v4/search?scope=blobs&search=log4j\" | jq\n\n  {\n    \"basename\": \"src/main/resources/log4j\",\n    \"data\": \"log4j.rootLogger=ERROR, stdout\\n \\n# Direct log messages to stdout\\n\",\n    \"path\": \"src/main/resources/log4j.properties\",\n    \"filename\": \"src/main/resources/log4j.properties\",\n    \"id\": null,\n    \"ref\": \"9a1df407e1a5365950a77f715163f6dba915fdf4\",\n    \"startline\": 2,\n    \"project_id\": 12345678\n  },\n\n```\n\n\nYou can use `jq` to further transform and filter the result set, for example\nonly listing the paths where `log4j` as a string exists.  \n\n\n```\n\ncurl --header \"PRIVATE-TOKEN: $GITLAB_TOKEN\"\n\"https://gitlab.com/api/v4/search?scope=blobs&search=log4j\" | jq -c '.[] |\nselect (.path | contains (\"log4j\"))' | jq\n\n```\n\n\n### Next steps \n\n\nThe GitLab security team is continuing to proactively monitor the situation\nand ensure our product and customers are secure. We will continue to\ncommunicate should we identify additional opportunities to help our\ncustomers and community navigate through this situation. Please [subscribe\nto our security alerts mailing\nlist](https://about.gitlab.com/company/preference-center/).\n\n\nPlease visit the public [log4j-resources\nproject](https://gitlab.com/gitlab-de/log4j-resources) and visit our\n[forum](https://forum.gitlab.com/c/devsecops-security/) for additional\ninformation.\n",[830,1241,9],{"slug":4569,"featured":6,"template":700},"use-gitlab-to-detect-vulnerabilities","content:en-us:blog:use-gitlab-to-detect-vulnerabilities.yml","Use Gitlab To Detect Vulnerabilities","en-us/blog/use-gitlab-to-detect-vulnerabilities.yml","en-us/blog/use-gitlab-to-detect-vulnerabilities",{"_path":4575,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4576,"content":4581,"config":4586,"_id":4588,"_type":13,"title":4589,"_source":15,"_file":4590,"_stem":4591,"_extension":18},"/en-us/blog/use-inputs-in-includable-files",{"title":4577,"description":4578,"ogTitle":4577,"ogDescription":4578,"noIndex":6,"ogImage":2157,"ogUrl":4579,"ogSiteName":685,"ogType":686,"canonicalUrls":4579,"schema":4580},"Define input parameters to includable CI/CD configuration files","This is the first milestone of the long-term roadmap of the CI/CD Components Catalog roadmap.","https://about.gitlab.com/blog/use-inputs-in-includable-files","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Define input parameters to includable CI/CD configuration files\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Itzik Gan Baruch\"}],\n        \"datePublished\": \"2023-05-08\",\n      }",{"title":4577,"description":4578,"authors":4582,"heroImage":2157,"date":4583,"body":4584,"category":785,"tags":4585},[1096],"2023-05-08","In GitLab 15.11, we introduced an exciting new feature that allows users to\n[define input parameters for includable configuration\nfiles](/releases/2023/04/22/gitlab-15-11-released/#define-inputs-for-included-cicd-configuration).\nWith the ability to use input parameters in [CI\ntemplates](https://docs.gitlab.com/ee/development/cicd/templates.html), you\ncan replace any keyword in the template with a parameter, including stage,\nscript, or job name. For example, you can add a prefix to all of the jobs to\nbetter isolate them from the pipeline into which you are including the\nconfiguration.\n\n\nThese input parameters can be declared as mandatory or optional for each\nconfiguration file, reducing the need for global variables and making your\nCI/CD templates more robust and isolated. The input parameters are scoped to\nthe included configuration only, which means they have no impact on the rest\nof the pipeline. This allows you to declare and enforce constraints, for\nexample by enforcing mandatory inputs for templates.\n\n\nThis development is the first milestone of the long-term roadmap of the\n[CI/CD Components\nCatalog](https://gitlab.com/groups/gitlab-org/-/epics/7462), a new feature\nthat will allow users to search and reuse single-purpose CI/CD configuration\nunits with specific parameters for their use case. If you want to learn more\nabout this exciting new development, you can read our [blog post about our\nCI templates feature](/blog/how-to-build-reusable-ci-templates/).\n\n\nIn this technical blog post, we will provide step-by-step instructions on\nhow to define CI/CD templates with input parameters and how to use them when\nincluding templates.\n\n\n## Step 1: Create a template YAML document\n\nThe first step is to create a template YAML document that describes what\ninput arguments can be used with the template. The second part of the\ntemplate is the definition of the jobs that may include references to values\nusing the interpolation format `$[[ inputs.input-name ]]`. You should use\nthree dash lines between the two parts.\n\n\nHere is an example of a deploy-template.yml:\n\n\n```yaml\n\nspec:\n  inputs:\n    website:\n    environment:\n      default: test\n---\n\ndeploy:\n  stage: deploy\n  script: echo \"deploy $[[ inputs.website ]] to $[[ inputs.environment ]]\"\n```\n\n\nIn this template, we have defined two input parameters: website and\nenvironment. The environment parameter has a default value. In the content\nsection, we define a job that interpolates the input arguments.\n\n\n## Step 2: Include the template in the CI configuration\n\nIn your main CI configuration file `.gitlab-ci.yml`, include the template\nand add input parameters using the `inputs` keyword.\n\n\nHere is an example of including the `deploy-template.yml` with input\nparameters:\n\n\n```yaml\n\ninclude:\n  - local: deploy-template.yml\n    inputs:\n      website: my-website.example.com\n```\n\n\nIn this example, we included a local template in our project. Note: You can\nuse `inputs` with the other [include\ntypes](https://docs.gitlab.com/ee/ci/yaml/index.html#include) such as\n`include:project`, `include:template`, `include:remote`.\n\n\nIn the below example, we use inputs to add a prefix to jobs name, and make\nthe stage dynamic as well.\n\n\n```yaml\n\nspec:\n  inputs:\n    website:\n    environment:\n      default: staging\n    stage:\n      default: test\n    job_prefix:\n      default: \"\"\n---\n\n\"$[[ inputs.job_prefix ]]deploy\":\n  stage: $[[ inputs.stage ]]\n  script: echo \"deploy $[[ inputs.website ]] to $[[ inputs.environment ]]\"\n```\n\n\nThen we can include it from the `.gitlab-ci.yml` with the input parameters:\n\n\n```\n\ninclude:\n  - local: deploy-template.yml\n    inputs:\n      stage: deploy\n      website: http://example.com\n      environment: production\n      job_prefix: \"my-app-\"\n```\n\n\nYou can [fork](https://gitlab.com/tech-marketing/ci-interpolation-example)\nthis project, which uses the above examples:\n\n\n- [Dynamic\njob](https://gitlab.com/tech-marketing/ci-interpolation-example/-/blob/main/dynamic-job.yml)\n\n- [Dynamic\nscript](https://gitlab.com/tech-marketing/ci-interpolation-example/-/blob/main/deploy-template.yml)\n\n- [Main CI\nconfiguration](https://gitlab.com/tech-marketing/ci-interpolation-example/-/blob/main/.gitlab-ci.yml)\n\n\nFor more information, please use our [online\ndocumentation](https://docs.gitlab.com/ee/ci/yaml/includes.html#define-input-parameters-with-specinputs).\n\n\nThat's it! You have successfully created CI templates that accept inputs and\nused them in a pipeline configuration. By using templates with inputs, you\ncan simplify pipeline configuration and make templates more modular and\nreusable.\n\n\nThank you to [Fabio Pitino](https://gitlab.com/fabiopitino) and [Grzegorz\nBizon](https://gitlab.com/grzesiek) for their content reviews.\n",[9,789,108],{"slug":4587,"featured":6,"template":700},"use-inputs-in-includable-files","content:en-us:blog:use-inputs-in-includable-files.yml","Use Inputs In Includable Files","en-us/blog/use-inputs-in-includable-files.yml","en-us/blog/use-inputs-in-includable-files",{"_path":4593,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4594,"content":4599,"config":4605,"_id":4607,"_type":13,"title":4608,"_source":15,"_file":4609,"_stem":4610,"_extension":18},"/en-us/blog/using-bazel-to-speed-up-gitlab-ci-builds",{"title":4595,"description":4596,"ogTitle":4595,"ogDescription":4596,"noIndex":6,"ogImage":1069,"ogUrl":4597,"ogSiteName":685,"ogType":686,"canonicalUrls":4597,"schema":4598},"How to use Bazel with GitLab to speed up your builds","We explain why Bazel and GitLab CI are a great match to speed up your build times.","https://about.gitlab.com/blog/using-bazel-to-speed-up-gitlab-ci-builds","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to use Bazel with GitLab to speed up your builds\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Jason Yavorska\"}],\n        \"datePublished\": \"2020-09-01\",\n      }",{"title":4595,"description":4596,"authors":4600,"heroImage":1069,"date":4602,"body":4603,"category":785,"tags":4604},[4601],"Jason Yavorska","2020-09-01","[Bazel](https://bazel.build/) is a useful tool that can be used with GitLab\nCI to push your build pipelines into overdrive.\n\n\nFor maximum correctness, [CI/CD](/topics/ci-cd/) systems will usually\nrebuild all of the artifacts from scratch on every run. This method is\nconsidered safer since artifacts from one pipeline won't negatively impact\nsubsequent pipelines, and is a lesson learned from older CI tools where the\nagent state was persistent over time – so you never really knew if you could\ndo a build from scratch. The problem with redoing everything every time\nthough, is that it's slow. GitLab improves upon this by using caches and\nshared artifacts, but there's only so far that approach can take you.\n\n\nBazel is a good example of tackling things in a different way – it speeds up\nbuilds by only rebuilding what is necessary. On the surface, this might\nsound a lot like just having a cache and doing an incremental build. But the\nmain difference is that Bazel is really good at not only being fast, but\nalso\n[correct](https://docs.bazel.build/versions/3.4.0/guide.html#correct-incremental-rebuilds).\nBazel is much more reliable than traditional `Makefiles` or build scripts,\nwhich are notorious for occasionally forcing you to `make clean` because\nthey get into some inconsistent state they can't recover from.\n\n\nAs of now, Bazel supports building Java, C, C++, Python, and Objective-C,\nand can also produce packages for deployment on Android or iOS. More\ncapabilities are being added all the time, as well as open source rule sets\nfor other languages like Go, Scala and many more, so be sure to check their\nlatest [product\noverview](https://docs.bazel.build/versions/3.4.0/bazel-overview.html) for\nupdates.\n\n\n## Setting up Bazel builds in GitLab CI\n\n\nSetting up Bazel for builds is very straightforward. A job like the\nfollowing does everything you need:\n\n\n```yaml\n\nvariables:\n  BAZEL_DIGEST_VERSION: \"f670e9aec235aa23a5f068566352c5850a67eb93de8d7a2350240c68fcec3b25\" # Bazel 3.4.1\n\nbuild:\n  image:\n    name: gcr.io/cloud-marketplace-containers/google/bazel@sha256:$BAZEL_DIGEST_VERSION\n    entrypoint: [\"\"]\n  stage: build\n  script:\n    - bazel --output_base output build //main/...\n  artifacts:\n    paths:\n      - bazel-bin/main/hello-world\n  cache:\n    key: $BAZEL_DIGEST_VERSION\n    paths:\n      - output\n```\n\n\nWhat this script does is define a job called `build` which uses the official\nGoogle Bazel image. We track the digest version for two reasons: First, to\nensure immutability (tags can be updated), and second to use it as a cache\nkey so that the cache is invalidated whenever we upgrade the Bazel version.\nWe also override the entry point because we want to pass our own parameters\nto our `bazel` invocation. The second parameter is the\n[label](https://docs.bazel.build/versions/master/glossary.html#label) of the\n[target](https://docs.bazel.build/versions/master/glossary.html#target) we\nwant to build. A [target\npattern](https://docs.bazel.build/versions/master/glossary.html#target-pattern)\ncan also be used here to tell Bazel to build multiple things (and what they\ndepend on), rather than one thing (and what it depends on).\n\n\nThe first parameter (`--output_base output`) is to help Bazel work with a\nsecurity feature of the GitLab runner. By default, the runner will [not\naccess files outside of the build\ndir](https://docs.gitlab.com/ee/ci/yaml/#artifactspaths), but Bazel places\nits own cache outside by default. This parameter tells Bazel to place it\ninside, where the runner can access it. The next two sections (`artifacts`\nand `cache`), tell the runner where the output file you want to keep is, and\nimportantly for Bazel, where the cache is that you want to persist. Note\nthat until [this issue to allow for traversing\nsymlinks](https://gitlab.com/gitlab-org/gitlab/-/issues/19746) is resolved,\nyou must give the full path to the specific outputs you want to keep within\nthe `bazel-bin` folder.\n\n\nWhen this job runs, it places the current cache (if it exists, and only for\nthe current `BAZEL_DIGEST_VERSION`) in the `output` folder, and then runs\n`bazel` to build the `main:hello-world` target. It saves the artifact from\n`bazel-bin/main/hello-world`, and then caches everything in `output` for the\nnext run.\n\n\n### Bazel: notes on caching\n\n\nIn this example we've set up Bazel to work with GitLab caching, and this is\nhow we currently use it internally. If you already have Bazel remote cache\n(or even better, Bazel remote execution), there is no need to set up GitLab\nCI cache: It actually would likely make things slower since in that case\nthere is no need to download and unpack the cache at all. Setting up remote\ncaching or remote execution are more advanced and outside of the scope of\nthis article, but are even better ways to speed up the build. Until then,\nusing a GitLab cache can be a good interim step. If you're interested in\nlearning more about remote cache/remote execution, this [BazelCon\nvideo](https://www.youtube.com/watch?v=MyuJRUwT5LI&t=1017s) or Bazel's\nofficial [documentation on remote\ncaching](https://docs.bazel.build/versions/master/remote-caching.html) may\nbe helpful.\n\n\n## Building and testing with Bazel\n\n\nUsing Bazel to run your tests is just as easy, and there are nice benefits\nto doing so. If you can rely on accurately knowing what has changed, you can\nbe more selective in doing incremental tests and have the confidence that\ntests that were skipped were truly unnecessary. This is also quite easy to\nset up using Bazel, but one thing to consider is that running builds and\ntests all at once (rather than splitting build and test into different jobs)\nis going to be more efficient. You can do that by using a build job that\nlooks like this:\n\n\n```yaml\n\nvariables:\n  BAZEL_DIGEST_VERSION: \"f670e9aec235aa23a5f068566352c5850a67eb93de8d7a2350240c68fcec3b25\" # 3.4.1\n\nbuild:\n  image:\n    name: gcr.io/cloud-marketplace-containers/google/bazel@sha256:$BAZEL_DIGEST_VERSION\n    entrypoint: [\"\"]\n  stage: build\n  script:\n    - bazel --output_base output test //main/...\n  artifacts:\n    paths:\n      - bazel-bin/main/hello-world\n  cache:\n    key: $BAZEL_DIGEST_VERSION\n    paths:\n      - output\n```\n\n\nIn a build that includes all tests, you typically want to run everything\nthat changed. That's usually done using an invocation like `bazel test\n//main/...` which:\n\n\n1. Finds all targets (referred to as `...`) in the workspace location (`//`\ndenotes the root of the\n[workspace](https://docs.bazel.build/versions/master/glossary.html#workspace)),\nso we are referring to `main` relative to the root.) Note that you probably\ndon't want to include a bare `//` (without `main`), since that will include\nthe custom `output` folder and that is probably not what you intended.\n\n1. Builds usual targets.\n\n1. Builds test targets.\n\n1. Runs test targets.\n\n\nOnly using the `test` parameter works because `bazel test` not only runs\ntests, but also builds everything that matched the target pattern by\ndefault. Individual targets can be excluded from being matched by `...` by\napplying a `manual` tag to them ([see `tags` in the Bazel glossary\ntable](https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes)).\nOne callout - in the example project we're building ([details\nbelow](#examples)), there actually aren't any tests, so this fails because\nwe requested a test pass and there weren't any. If your project has tests in\nit, it will work fine.\n\n\n## Examples using Bazel\n\n\nWe're actually using Bazel here at GitLab to build our [GitLab Agent for\nKubernetes](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent).\nIf you're interested in seeing a more complex, complete implementation using\nBazel then that's a great one to explore. The simple example from this blog\ncan be found live in [my own personal\nproject](https://gitlab.com/jyavorska/testbazel), and it is based on the\n[stage three build\ntutorial](https://docs.bazel.build/versions/3.4.0/tutorial/cpp.html) from\nBazel's own documentation.\n\n\nBazel itself is also highly configurable through its own `.bazelrc`, `BUILD`\nfiles, and more. The [user documentation for\nBazel](https://docs.bazel.build/versions/master/guide.html) contains several\nexamples along with an exhaustive configuration reference.\n\n\n## What's next with Bazel?\n\n\nWe are considering using Bazel in few more areas within GitLab:\n\n\n- In an ideal world, after a minor change, the build and test should only\ntake a few seconds to complete. When the jobs are fast enough, it could even\nbe triggered via an editor on every change before being committed to git at\nall. This kind of capability could be integrated with the Web IDE, giving\nyou immediate insight into the results of your change. We have an issue\nrelated to [making it easier to run pipelines from the Web\nIDE](https://gitlab.com/gitlab-org/gitlab/-/issues/213604) that could take\nadvantage of this.\n\n- By default, GitLab uses [a gem we\ncreated](https://gitlab.com/gitlab-org/ci-cd/test_file_finder/) (which is\navailable in this\n[template](https://docs.gitlab.com/ee/ci/testing/fail_fast_testing.html) for\ntest execution optimization, but all we're doing so far is running the\nriskiest tests first. As Bazel grows and adds support for more languages, it\ncould potentially become a standard for this purpose, allowing you to run\neven fewer tests (and among those, the riskiest ones first). We have an\n[epic](https://gitlab.com/groups/gitlab-org/-/epics/4121) where you can\ntrack progress toward this idea.\n\n- Finally, Bazel also supports distributed builds and caching, opening the\ndoor to autoscaling compilation and test capacity alongside runner capacity,\nor even sharing the same capacity for whatever jobs are needed at a given\nmoment. This function would require managing your own capacity for this\npurpose, but in the future we could imagine this being added to GitLab. We\nhave an [issue for exploring different ways Bazel could support distributed\njobs](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26663) using the\nGitLab Runner.\n\n\n## Tell us your Bazel success stories\n\n\nAre you using Bazel with GitLab CI? We'd love your feedback on what features\nwe could add to make things work better and hear about the performance gains\nyou've found from the combo. Please let us know in the Meta issue below, or\ncontact [Jason Yavorska](https://twitter.com/j4yav) on Twitter.\n\n\n## Related content\n\n\n- [Bazel website](https://bazel.build/)\n\n- [Meta issue for deeper integration in\nGitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/201484)\n\n- [Bazel blog on integrating it with CI\nsystems](https://blog.bazel.build/2016/01/27/continuous-integration.html)\n\n- [GitLab CI quick start](https://docs.gitlab.com/ee/ci/quick_start/)\n\n\nCover image by [Lucas van Oort](https://unsplash.com/@switch_dtp_fotografie)\non [Unsplash](https://unsplash.com)\n\n{: .note}\n",[108,232,9],{"slug":4606,"featured":6,"template":700},"using-bazel-to-speed-up-gitlab-ci-builds","content:en-us:blog:using-bazel-to-speed-up-gitlab-ci-builds.yml","Using Bazel To Speed Up Gitlab Ci Builds","en-us/blog/using-bazel-to-speed-up-gitlab-ci-builds.yml","en-us/blog/using-bazel-to-speed-up-gitlab-ci-builds",{"_path":4612,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4613,"content":4619,"config":4625,"_id":4627,"_type":13,"title":4628,"_source":15,"_file":4629,"_stem":4630,"_extension":18},"/en-us/blog/using-child-pipelines-to-continuously-deploy-to-five-environments",{"title":4614,"description":4615,"ogTitle":4614,"ogDescription":4615,"noIndex":6,"ogImage":4616,"ogUrl":4617,"ogSiteName":685,"ogType":686,"canonicalUrls":4617,"schema":4618},"Using child pipelines to continuously deploy to five environments","Learn how to manage continuous deployment to multiple environments, including temporary, on-the-fly sandboxes, with a minimalist GitLab workflow.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097012/Blog/Hero%20Images/Blog/Hero%20Images/AdobeStock_397632156_3Ldy1urjMStQCl4qnOBvE0_1750097011626.jpg","https://about.gitlab.com/blog/using-child-pipelines-to-continuously-deploy-to-five-environments","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Using child pipelines to continuously deploy to five environments\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Olivier Dupré\"}],\n        \"datePublished\": \"2024-09-26\",\n      }",{"title":4614,"description":4615,"authors":4620,"heroImage":4616,"date":4622,"body":4623,"category":785,"tags":4624},[4621],"Olivier Dupré","2024-09-26","DevSecOps teams sometimes require the ability to manage continuous\ndeployment across multiple environments — and they need to do so without\nchanging their workflows. The [GitLab DevSecOps\nplatform](https://about.gitlab.com/) supports this need, including\ntemporary, on-the-fly sandboxes, with a minimalist approach. In this\narticle, you'll learn how to run continuous deployment of infrastructure\nusing Terraform, over multiple environments.\n\n\nThis strategy can easily be applied to any project, whether it is\ninfrastructure as code (IaC) relying on another technology, such as\n[Pulumi](https://www.pulumi.com/) or [Ansible](https://www.ansible.com/),\nsource code in any language, or a monorepo that mixes many languages.\n\n\nThe final pipeline that you will have at the end of this tutorial will\ndeploy:\n\n\n* A temporary **review** environment for each feature branch.\n\n* An **integration** environment, easy to wipe out and deployed from the\nmain branch.\n\n* A **QA** environment, also deployed from the main branch, to run quality\nassurance steps.\n\n* A **staging** environment, deployed for every tag. This is the last round\nbefore production.\n\n* A **production** environment, just after the staging environment. This one\nis triggered manually for demonstration, but can also be continuously\ndeployed.\n\n\n>Here is the legend for the flow charts in this article:\n\n> * Round boxes are the GitLab branches.\n\n> * Square boxes are the environments.\n\n> * Text on the arrows are the actions to flow from one box to the next.\n\n> * Angled squares are decision steps.\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    A(main) -->|new feature| B(feature_X)\n\n    B -->|auto deploy| C[review/feature_X]\n    B -->|merge| D(main)\n    C -->|destroy| D\n\n    D -->|auto deploy| E[integration]\n    E -->|manual| F[qa]\n\n    D -->|tag| G(X.Y.Z)\n    F -->|validate| G\n\n    G -->|auto deploy| H[staging]\n    H -->|manual| I{plan}\n    I -->|manual| J[production]\n\u003C/pre>\n\n\nOn each step, you'll learn the [why](#why) and the [what](#what) before\nmoving to the [how](#how). This will help you fully understand and replicate\nthis tutorial.\n\n\n## Why\n\n\n* [Continuous\nintegration](https://about.gitlab.com/topics/ci-cd/#what-is-continuous-integration-ci)\nis almost a de facto standard. Most companies have implemented CI pipelines\nor are willing to standardize their practice.\n\n\n* [Continuous\ndelivery](https://about.gitlab.com/topics/ci-cd/#what-is-continuous-delivery-cd),\nwhich pushes artifacts to a repository or registry at the end of the CI\npipeline, is also popular.\n\n\n* Continuous deployment, which goes further and deploys these artifacts\nautomatically, is less widespread. When it has been implemented, we see it\nessentially in the application field. When discussing continuously\ndeploying  infrastructure, the picture seems less obvious, and is more about\nmanaging several environments. In contrast, testing, securing, and verifying\nthe infrastructure's code seems more challenging. And this is one of the\nfields where DevOps has not yet reached its maturity. One of the other\nfields is to shift security left, integrating security teams and, more\nimportantly, security concerns, earlier in the delivery lifecycle, to\nupgrade from DevOps to ***DevSecOps***.\n\n\nGiven this high-level picture, in this tutorial, you will work toward a\nsimple, yet efficient way to implement DevSecOps for your infrastructure\nthrough the example of deploying resources to five environments, gradually\nprogressing from development to production.\n\n\n__Note:__ Even if I advocate embracing a FinOps approach and reducing the\nnumber of environments, sometimes there are excellent reasons to maintain\nmore than just dev, staging, and production. So, please, adapt the examples\nbelow to match your needs.\n\n\n## What\n\n\nThe rise of cloud technology has driven the usage of IaC. Ansible and\nTerraform were among the first to pave the road here. OpenTofu, Pulumi, AWS\nCDK, Google Deploy Manager, and many others joined the party.\n\n\nDefining IaC is a perfect solution to feel safe when deploying\ninfrastructure. You can test it, deploy it, and replay it again and again\nuntil you reach your goal.\n\n\nUnfortunately, we often see companies maintain several branches, or even\nrepositories, for each of their target environments. And this is where the\nproblems start. They are no longer enforcing a process. They are no longer\nensuring that any change in the production code base has been accurately\ntested in previous environments. And they start seeing drifts from one\nenvironment to the other.\n\n\nI realized this tutorial was necessary when, at a conference I attended,\nevery participant said they do not have a workflow that enforces the\ninfrastructure to be tested thoroughly before being deployed to production.\nAnd they all agreed that sometimes they patch the code directly in\nproduction. Sure, this is fast, but is it safe? How do you report back to\nprevious environments? How do you ensure there are no side effects? How do\nyou control whether you are putting your company at risk with new\nvulnerabilities being pushed too quickly in production?\n\n\nThe question of *why* DevOps teams deploy directly to production is critical\nhere. Is it because the pipeline could be more efficient or faster? Is there\nno automation? Or, even worse, because there is *no way to test accurately\noutside of production*?\n\n\nIn the next section, you will learn how to implement automation for your\ninfrastructure and ensure that your DevOps team can effectively test what\nyou are doing before pushing to any environment impacting others. You will\nsee how your code is secured and its deployment is controlled, end-to-end.\n\n\n## How\n\n\nAs mentioned earlier, there are many IaC languages out there nowadays and we\nobjectively cannot cover *all* of them in a single article. So, I will rely\non a basic Terraform code running on Version 1.4. Please do not focus on the\nIaC language itself but instead on the process that you could apply to your\nown ecosystem.\n\n\n### The Terraform code\n\n\nLet's start with a fundamental Terraform code.\n\n\nWe are going to deploy to AWS, a virtual private cloud (VPC), which is a\nvirtual network. In that VPC, we will deploy a public and a private subnet.\nAs their name implies, they are subnets of the main VPC. Finally, we will\nadd an Elastic Cloud Compute (EC2) instance (a virtual machine) in the\npublic subnet.\n\n\nThis demonstrates the deployment of four resources without adding too much\ncomplexity. The idea is to focus on the pipeline, not the code.\n\n\nHere is the target we want to reach for your repository.\n\n\n![target for\nrepository](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097033/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097033415.png)\n\n\nLet’s do it step by step.\n\n\nFirst, we declare all resources in a `terraform/main.tf` file:\n\n\n```terraform\n\nprovider \"aws\" {\n  region = var.aws_default_region\n}\n\n\nresource \"aws_vpc\" \"main\" {\n  cidr_block = var.aws_vpc_cidr\n\n  tags = {\n    Name     = var.aws_resources_name\n  }\n}\n\n\nresource \"aws_subnet\" \"public_subnet\" {\n  vpc_id     = aws_vpc.main.id\n  cidr_block = var.aws_public_subnet_cidr\n\n  tags = {\n    Name = \"Public Subnet\"\n  }\n}\n\nresource \"aws_subnet\" \"private_subnet\" {\n  vpc_id     = aws_vpc.main.id\n  cidr_block = var.aws_private_subnet_cidr\n\n  tags = {\n    Name = \"Private Subnet\"\n  }\n}\n\n\nresource \"aws_instance\" \"sandbox\" {\n  ami           = var.aws_ami_id\n  instance_type = var.aws_instance_type\n\n  subnet_id = aws_subnet.public_subnet.id\n\n  tags = {\n    Name     = var.aws_resources_name\n  }\n}\n\n```\n\n\nAs you can see, there are a couple of variables that are needed for this\ncode, so let's declare them in a `terraform/variables.tf` file:\n\n\n```terraform\n\nvariable \"aws_ami_id\" {\n  description = \"The AMI ID of the image being deployed.\"\n  type        = string\n}\n\n\nvariable \"aws_instance_type\" {\n  description = \"The instance type of the VM being deployed.\"\n  type        = string\n  default     = \"t2.micro\"\n}\n\n\nvariable \"aws_vpc_cidr\" {\n  description = \"The CIDR of the VPC.\"\n  type        = string\n  default     = \"10.0.0.0/16\"\n}\n\n\nvariable \"aws_public_subnet_cidr\" {\n  description = \"The CIDR of the public subnet.\"\n  type        = string\n  default     = \"10.0.1.0/24\"\n}\n\n\nvariable \"aws_private_subnet_cidr\" {\n  description = \"The CIDR of the private subnet.\"\n  type        = string\n  default     = \"10.0.2.0/24\"\n}\n\n\nvariable \"aws_default_region\" {\n  description = \"Default region where resources are deployed.\"\n  type        = string\n  default     = \"eu-west-3\"\n}\n\n\nvariable \"aws_resources_name\" {\n  description = \"Default name for the resources.\"\n  type        = string\n  default     = \"demo\"\n}\n\n```\n\n\nAlready, we are almost good to go on the IaC side. What's missing is a way\nto share the Terraform states. For those who don't know, Terraform works\nschematically doing the following:\n\n\n* `plan` checks the differences between the current state of the\ninfrastructure and what is defined in the code. Then, it outputs the\ndifferences.\n\n* `apply` applies the differences in the `plan` and updates the state.\n\n\nFirst round, the state is empty, then it is filled with the details (ID,\netc.) of the resources applied by Terraform.\n\n\nThe problem is: Where is that state stored? How do we share it so several\ndevelopers can collaborate on code?\n\n\nThe solution is fairly simple: Leverage GitLab to store and share the state\nfor you through a [Terraform HTTP\nbackend](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html).\n\n\nThe first step in using this backend is to create the most simple\n`terraform/backend.tf` file. The second step will be handled in the\npipeline.\n\n\n```terraform\n\nterraform {\n  backend \"http\" {\n  }\n}\n\n```\n\n\nEt voilà! We have a bare minimum Terraform code to deploy these four\nresources. We will provide the variable values at the runtime, so let's do\nthat later.\n\n\n### The workflow\n\n\nThe workflow that we are going to implement now is the following:\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    A(main) -->|new feature| B(feature_X)\n\n    B -->|auto deploy| C[review/feature_X]\n    B -->|merge| D(main)\n    C -->|destroy| D\n\n    D -->|auto deploy| E[integration]\n    E -->|manual| F[qa]\n\n    D -->|tag| G(X.Y.Z)\n    F -->|validate| G\n\n    G -->|auto deploy| H[staging]\n    H -->|manual| I{plan}\n    I -->|manual| J[production]\n\u003C/pre>\n\n\n1. Create a **feature** branch. This will continuously run all scanners on\nthe code to ensure that it is still compliant and secured. This code will be\ncontinuously deployed to a temporary environment `review/feature_branch`\nwith the name of the current branch. This is a safe environment where the\ndevelopers and operations teams can test their code without impacting\nanybody. This is also where we will enforce the process, like enforcing code\nreviews and running scanners, to ensure that the quality and security of the\ncode are acceptable and do not put your assets at risk. The infrastructure\ndeployed by this branch is automatically destroyed when the branch is\nclosed. This helps you keep your budget under control.\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    A(main) -->|new feature| B(feature_X)\n\n    B -->|auto deploy| C[review/feature_X]\n    B -->|merge| D(main)\n    C -->|destroy| D\n\u003C/pre>\n\n\n2. Once approved, the feature branch will be **merged** into the main\nbranch. This is a [protected\nbranch](https://docs.gitlab.com/ee/user/project/protected_branches.html)\nwhere no one can push. This is mandatory to ensure that every change request\nto production is thoroughly tested. That branch is also continuously\ndeployed. The target here is the `integration` environment. To keep this\nenvironment slightly more stable, its deletion is not automated but can be\ntriggered manually.\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    D(main) -->|auto deploy| E[integration]\n\u003C/pre>\n\n\n3. From there, manual approval is required to trigger the next deployment.\nThis will deploy the main branch to the `qa` environment. Here, I have set a\nrule to prevent deletion from the pipeline. The idea is that this\nenvironment should be quite stable (after all, it's already the third\nenvironment), and I would like to prevent deletion by mistake. Feel free to\nadapt the rules to match your processes.\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    D(main)-->|auto deploy| E[integration]\n    E -->|manual| F[qa]\n\u003C/pre>\n\n\n4. To proceed, we will need to **tag** the code. We are relying on\n[protected\ntags](https://docs.gitlab.com/ee/user/project/protected_tags.html) here to\nensure that only a specific set of users are allowed to deploy to these last\ntwo environments. This will immediately trigger a deployment to the\n`staging` environment.\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    D(main) -->|tag| G(X.Y.Z)\n    F[qa] -->|validate| G\n\n    G -->|auto deploy| H[staging]\n\u003C/pre>\n\n\n5. Finally, we are landing to `production`. When discussing infrastructure,\nit is often challenging to deploy progressively (10%, 25%, etc.), so we will\ndeploy the whole infrastructure. Still, we control that deployment with a\nmanual trigger of this last step. And to enforce maximum control on this\nhighly critical environment, we will control it as a [protected\nenvironment](https://docs.gitlab.com/ee/ci/environments/protected_environments.html).\n\n\n\u003Cpre class=\"mermaid\">\n\nflowchart LR\n    H[staging] -->|manual| I{plan}\n    I -->|manual| J[production]\n\u003C/pre>\n\n\n### The pipeline\n\n\nTo implement the above [workflow](#the-workflow), we are now going to\nimplement a pipeline with two [downstream\npipelines](https://docs.gitlab.com/ee/ci/pipelines/downstream_pipelines.html).\n\n\n#### The main pipeline\n\n\nLet's start with the main pipeline. This is the one that will be triggered\nautomatically on any **push to a feature branch**, any **merge to the\ndefault branch**, or any **tag**. *The one* that will do true **continuous\ndeployment** to the following environments: `dev`, `integration`, and\n`staging`. And it is declared in the `.gitlab-ci.yml` file at the root of\nyour project.\n\n\n![the repository\ntarget](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097033/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097033417.png)\n\n\n```yml\n\nStages:\n  - test\n  - environments\n\n.environment:\n  stage: environments\n  variables:\n    TF_ROOT: terraform\n    TF_CLI_ARGS_plan: \"-var-file=../vars/$variables_file.tfvars\"\n  trigger:\n    include: .gitlab-ci/.first-layer.gitlab-ci.yml\n    strategy: depend            # Wait for the triggered pipeline to successfully complete\n    forward:\n      yaml_variables: true      # Forward variables defined in the trigger job\n      pipeline_variables: true  # Forward manual pipeline variables and scheduled pipeline variables\n\nreview:\n  extends: .environment\n  variables:\n    environment: review/$CI_COMMIT_REF_SLUG\n    TF_STATE_NAME: $CI_COMMIT_REF_SLUG\n    variables_file: review\n    TF_VAR_aws_resources_name: $CI_COMMIT_REF_SLUG  # Used in the tag Name of the resources deployed, to easily differenciate them\n  rules:\n    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n\nintegration:\n  extends: .environment\n  variables:\n    environment: integration\n    TF_STATE_NAME: $environment\n    variables_file: $environment\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n\nstaging:\n  extends: .environment\n  variables:\n    environment: staging\n    TF_STATE_NAME: $environment\n    variables_file: $environment\n  rules:\n    - if: $CI_COMMIT_TAG\n\n#### TWEAK\n\n# This tweak is needed to display vulnerability results in the merge\nwidgets.\n\n# As soon as this issue https://gitlab.com/gitlab-org/gitlab/-/issues/439700\nis resolved, the `include` instruction below can be removed.\n\n# Until then, the SAST IaC scanners will run in the downstream pipelines,\nbut their results will not be available directly in the merge request\nwidget, making it harder to track them.\n\n# Note: This workaround is perfectly safe and will not slow down your\npipeline.\n\ninclude:\n  - template: Security/SAST-IaC.gitlab-ci.yml\n#### END TWEAK\n\n\n```\n\n\nThis pipeline runs only two stages: `test` and  `environments`. The former\nis needed for the *TWEAK* to run scanners. The later triggers a child\npipeline with a different set of variables for each case defined above (push\nto the branch, merge to the default branch, or tag).\n\n\nWe are adding here a dependency with the keyword\n[strategy:depend](https://docs.gitlab.com/ee/ci/yaml/index.html#triggerstrategy)\non our child pipeline so the pipeline view in GitLab will be updated only\nonce the deployment is finished.\n\n\nAs you can see here, we are defining a base job,\n[hidden](https://docs.gitlab.com/ee/ci/jobs/#hide-jobs), and we are\nextending it with specific variables and rules to trigger only one\ndeployment for each target environment.\n\n\nBesides the [predefined\nvariables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html),\nwe are using two new entries that we need to define:\n\n1. [The variables specific](#the-variable-definitions) to each environment:\n`../vars/$variables_file.tfvars`\n\n2. [The child pipeline](#the-child-pipeline), defined in\n`.gitlab-ci/.first-layer.gitlab-ci.yml`\n\n\nLet's start with the smallest part, the variable definitions.\n\n\n### The variable definitions\n\n\nWe are going here to mix two solutions to provide variables to Terraform:\n\n\n* The first one using [.tfvars\nfiles](https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files)\nfor all non-sensitive input, which should be stored within GitLab.\n\n\n![solution one to provide variables to\nTerraform](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097034/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097033419.png)\n\n\n* The second using [environment\nvariables](https://developer.hashicorp.com/terraform/language/values/variables#environment-variables)\nwith the prefix `TF_VAR`. That second way to inject variables, associated\nwith the GitLab capacity to [mask\nvariables](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable),\n[protect\nthem](https://docs.gitlab.com/ee/ci/variables/#protect-a-cicd-variable), and\n[scope them to\nenvironments](https://docs.gitlab.com/ee/ci/environments/index.html#limit-the-environment-scope-of-a-cicd-variable)\nis a powerful solution to **prevent sensitive information leakages**. (If\nyou consider your production’s private CIDR very sensitive, you could\nprotect it like this, ensuring it is only available for the `production`\nenvironment, for pipelines running against protected branches and tags, and\nthat its value is masked in the job’s logs.)\n\n\n![solution two to provide variables to\nTerraform](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097034/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097033422.png)\n\n\nAdditionally, each variable file should be controlled through a\n[`CODEOWNERS` file](https://docs.gitlab.com/ee/user/project/codeowners/) to\nset who can modify each of them.\n\n\n```\n\n[Production owners] \n\nvars/production.tfvars @operations-group\n\n\n[Staging owners]\n\nvars/staging.tfvars @odupre @operations-group\n\n\n[CodeOwners owners]\n\nCODEOWNERS @odupre\n\n```\n\n\nThis article is not a Terraform training, so we will go very fast and simply\nshow here the `vars/review.tfvars` file. Subsequent environment files are,\nof course, very similar. Just set the non-sensitive variables and their\nvalues here.\n\n\n```shell\n\naws_vpc_cidr = \"10.1.0.0/16\"\n\naws_public_subnet_cidr = \"10.1.1.0/24\"\n\naws_private_subnet_cidr = \"10.1.2.0/24\"\n\n```\n\n\n#### The child pipeline\n\n\nThis one is where the actual work is done. So, it is slightly more complex\nthan the first one. But there is no difficulty here that we cannot overcome\ntogether!\n\n\nAs we have seen in the definition of the [main\npipeline](#the-main-pipeline), that downstream pipeline is declared in the\nfile `.gitlab-ci/.first-layer.gitlab-ci.yml`.\n\n\n![Downstream pipeline declared in\nfile](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097033/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097033424.png)\n\n\nLet's break it down into small chunks. We'll see the big picture at the end.\n\n\n##### Run Terraform commands and secure the code\n\n\nFirst, we want to run a pipeline for Terraform. We, at GitLab, are open\nsource. So, our Terraform template is open source. And you simply need to\ninclude it. This can be achieved with the following snippet:\n\n\n```yml\n\ninclude:\n  - template: Terraform.gitlab-ci.yml\n```\n\n\nThis template runs for you the Terraform checks on the formatting and\nvalidates your code, before planning and applying it. It also allows you to\ndestroy what you have deployed.\n\n\nAnd, because GitLab is the a single, unified DevSecOps platform, we are also\nautomatically including two security scanners within that template to find\npotential threats in your code and warn you before you deploy it to the next\nenvironments.\n\n\nNow that we have checked, secured, built, and deployed our code, let's do\nsome tricks.\n\n\n##### Share cache between jobs\n\n\nWe will cache the job results to reuse them in subsequent pipeline jobs.\nThis is as simple as adding the following piece of code:\n\n\n```yml\n\ndefault:\n  cache:  # Use a shared cache or tagged runners to ensure terraform can run on apply and destroy\n    - key: cache-$CI_COMMIT_REF_SLUG\n      fallback_keys:\n        - cache-$CI_DEFAULT_BRANCH\n      paths:\n        - .\n```\n\n\nHere, we are defining a different cache for each commit, falling back to the\nmain branch name if needed.\n\n\nIf we look carefully at the templates that we are using, we can see that it\nhas some rules to control when jobs are run. We want to run all controls\n(both QA and security) on all branches. So, we are going to override these\nsettings.\n\n\n##### Run controls on all branches\n\n\nGitLab templates are a powerful feature where one can override only a piece\nof the template. Here, we are interested only in overwriting the rules of\nsome jobs to always run quality and security checks. Everything else defined\nfor these jobs will stay as defined in the template.\n\n\n```yml\n\nfmt:\n  rules:\n    - when: always\n\nvalidate:\n  rules:\n    - when: always\n\nkics-iac-sast:\n  rules:\n    - when: always\n\niac-sast:\n  rules:\n    - when: always\n```\n\n\nNow that we have enforced the quality and security controls, we want to\ndifferentiate how the main environments (integration and staging) in the\n[workflow](#the-workflow) and review environments behave. Let's start by\ndefining the main environment’s behavior, and we will tweak this\nconfiguration for the review environments.\n\n\n##### CD to integration and staging\n\n\nAs defined earlier, we want to deploy the main branch and the tags to these\ntwo environments. We are adding rules to control that on both the `build`\nand `deploy` jobs. Then, we want to enable `destroy` only for the\n`integration` as we have defined `staging` to be too critical to be deleted\nwith a single click. This is error-prone and we don't want to do that.\n\n\nFinally, we are linking the `deploy` job to the `destroy` one, so we can\n`stop` the environment directly from GitLab GUI.\n\n\nThe `GIT_STRATEGY` is here to prevent retrieving the code from the source\nbranch in the runner when destroying. This would fail if the branch has been\ndeleted manually, so we are relying on the cache to get everything we need\nto run the Terraform instructions.\n\n\n```yml\n\nbuild:  # terraform plan\n  environment:\n    name: $TF_STATE_NAME\n    action: prepare\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    - if: $CI_COMMIT_TAG\n\ndeploy: # terraform apply --> automatically deploy on corresponding env\n(integration or staging) when merging to default branch or tagging. Second\nlayer environments (qa and production) will be controlled manually\n  environment: \n    name: $TF_STATE_NAME\n    action: start\n    on_stop: destroy\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    - if: $CI_COMMIT_TAG\n\ndestroy:\n  extends: .terraform:destroy\n  variables:\n    GIT_STRATEGY: none\n  dependencies:\n    - build\n  environment:\n    name: $TF_STATE_NAME\n    action: stop\n  rules:\n    - if: $CI_COMMIT_TAG  # Do not destroy production\n      when: never\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_DESTROY == \"true\" # Manually destroy integration env.\n      when: manual\n```\n\n\nAs said, this matches the need to deploy to `integration` and `staging`. But\nwe are still missing a temporary environment where the developers can\nexperience and validate their code without impacts on others. This is where\nthe deployment to the `review` environment takes place.\n\n\n##### CD to review environments\n\n\nDeploying to review environment is not too different than deploying to\n`integration` and `staging`. So we will once again leverage GitLab's\ncapacity to overwrite only pieces of job definition here.\n\n\nFirst, we set rules to run these jobs only on feature branches.\n\n\nThen, we link the `deploy_review` job to `destroy_review`. This will allow\nus to stop the environment **manually** from the GitLab user interface, but\nmore importantly, it will **automatically trigger the environment\ndestruction** when the feature branch is closed. This is a good FinOps\npractice to help you control your operational expenditures.\n\n\nSince Terraform needs a plan file to destroy an infrastructure, exactly like\nit needs one to build an infrastructure, then we are adding a dependency\nfrom `destroy_review` to `build_review`, to retrieve its artifacts.\n\n\nFinally, we see here that the environment's name is set to `$environment`.\nIt has been set in the [main pipeline](#the-main-pipeline) to\n`review/$CI_COMMIT_REF_SLUG`, and forwarded to this child pipeline with the\ninstruction `trigger:forward:yaml_variables:true`.\n\n\n```yml\n\nbuild_review:\n  extends: build\n  rules:\n    - if: $CI_COMMIT_TAG\n      when: never\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n      when: on_success\n\ndeploy_review:\n  extends: deploy\n  dependencies:\n    - build_review\n  environment:\n    name: $environment\n    action: start\n    on_stop: destroy_review\n    # url: https://$CI_ENVIRONMENT_SLUG.example.com\n  rules:\n    - if: $CI_COMMIT_TAG\n      when: never\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n      when: on_success\n\ndestroy_review:\n  extends: destroy\n  dependencies:\n    - build_review\n  environment:\n    name: $environment\n    action: stop\n  rules:\n    - if: $CI_COMMIT_TAG  # Do not destroy production\n      when: never\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH   # Do not destroy staging\n      when: never\n    - when: manual\n```\n\n\nSo, to recap, we now have a pipeline that can:\n\n\n* Deploy temporary review environments, which are automatically cleaned up\nwhen the feature branch is closed\n\n* Continuously deploy the **default branch** to `integration`\n\n* Continuously deploy the **tags** to `staging`\n\n\nLet's now add an extra layer, where we will deploy, based on a manual\ntrigger this time, to `qa` and `production` environments.\n\n\n##### Continously deploy to QA and production\n\n\nBecause not everybody is willing to deploy continuously to production, we\nwill add a manual validation to the next two deployments. From a purely\n**CD** perspective, we would not add this trigger, but take this as an\nopportunity to learn how to run jobs from other triggers.\n\n\nSo far, we have started a [child pipeline](#the-child-pipeline) from the\n[main pipeline](#the-main-pipeline) to run all deployments.\n\n\nSince we want to run other deployments from the default branch and the tags,\nwe will add another layer dedicated to these additional steps. Nothing new\nhere. We will just repeat exactly the same process as the one we only did\nfor the [main pipeline](#the-main-pipeline). Going this way allows you to\nmanipulate as many layers as you need. I have already seen up to nine\nenvironments in some places.\n\n\nWithout arguing once again on the benefits to have fewer environments, the\nprocess that we are using here makes it very easy to implement the same\npipeline all the way from early stages to final delivery, while keeping your\npipeline definition simple and split in small chunks that you can maintain\nat no cost.\n\n\nTo prevent variable conflicts here, we are just using new var names to\nidentify the Terraform state and input file.\n\n\n```yml\n\n.2nd_layer:\n  stage: 2nd_layer\n  variables:\n    TF_ROOT: terraform\n  trigger:\n    include: .gitlab-ci/.second-layer.gitlab-ci.yml\n    # strategy: depend            # Do NOT wait for the downstream pipeline to finish to mark upstream pipeline as successful. Otherwise, all pipelines will fail when reaching the pipeline timeout before deployment to 2nd layer.\n    forward:\n      yaml_variables: true      # Forward variables defined in the trigger job\n      pipeline_variables: true  # Forward manual pipeline variables and scheduled pipeline variables\n\nqa:\n  extends: .2nd_layer\n  variables:\n    TF_STATE_NAME_2: qa\n    environment: $TF_STATE_NAME_2\n    TF_CLI_ARGS_plan_2: \"-var-file=../vars/$TF_STATE_NAME_2.tfvars\"\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n\nproduction:\n  extends: .2nd_layer\n  variables:\n    TF_STATE_NAME_2: production\n    environment: $TF_STATE_NAME_2\n    TF_CLI_ARGS_plan_2: \"-var-file=../vars/$TF_STATE_NAME_2.tfvars\"\n  rules:\n    - if: $CI_COMMIT_TAG\n```\n\n\n**One important trick here is the strategy used for the new downstream\npipeline.** We leave that `trigger:strategy` to its default value;\notherwise, the [main pipeline](#the-main-pipeline) would wait for your\n[grand-child pipeline](#the-grand-child-pipeline) to finish. With a manual\ntrigger, this could last for a very long time and make your pipeline\ndashboard harder to read and understand.\n\n\nYou have probably already wondered what is the content of that\n`.gitlab-ci/.second-layer.gitlab-ci.yml` file we are including here.  We\nwill cover that in the next section.\n\n\n##### The first layer complete pipeline definition\n\n\nIf you are looking for a complete view of this first layer (stored in\n`.gitlab-ci/.first-layer.gitlab-ci.yml`), just expand the section below.\n\n\n```yml\n\nvariables:\n  TF_VAR_aws_ami_id: $AWS_AMI_ID\n  TF_VAR_aws_instance_type: $AWS_INSTANCE_TYPE\n  TF_VAR_aws_default_region: $AWS_DEFAULT_REGION\n\ninclude:\n  - template: Terraform.gitlab-ci.yml\n\ndefault:\n  cache:  # Use a shared cache or tagged runners to ensure terraform can run on apply and destroy\n    - key: cache-$CI_COMMIT_REF_SLUG\n      fallback_keys:\n        - cache-$CI_DEFAULT_BRANCH\n      paths:\n        - .\n\nstages:\n  - validate\n  - test\n  - build\n  - deploy\n  - cleanup\n  - 2nd_layer       # Use to deploy a 2nd environment on both the main branch and on the tags\n\nfmt:\n  rules:\n    - when: always\n\nvalidate:\n  rules:\n    - when: always\n\nkics-iac-sast:\n  rules:\n    - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'\n      when: never\n    - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/\n      when: never\n    - when: on_success\n\niac-sast:\n  rules:\n    - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'\n      when: never\n    - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/\n      when: never\n    - when: on_success\n\n###########################################################################################################\n\n## Integration env. and Staging. env\n\n##  * Auto-deploy to Integration on merge to main.\n\n##  * Auto-deploy to Staging on tag.\n\n##  * Integration can be manually destroyed if TF_DESTROY is set to true.\n\n##  * Destroy of next env. is not automated to prevent errors.\n\n###########################################################################################################\n\nbuild:  # terraform plan\n  environment:\n    name: $TF_STATE_NAME\n    action: prepare\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    - if: $CI_COMMIT_TAG\n\ndeploy: # terraform apply --> automatically deploy on corresponding env\n(integration or staging) when merging to default branch or tagging. Second\nlayer environments (qa and production) will be controlled manually\n  environment: \n    name: $TF_STATE_NAME\n    action: start\n    on_stop: destroy\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    - if: $CI_COMMIT_TAG\n\ndestroy:\n  extends: .terraform:destroy\n  variables:\n    GIT_STRATEGY: none\n  dependencies:\n    - build\n  environment:\n    name: $TF_STATE_NAME\n    action: stop\n  rules:\n    - if: $CI_COMMIT_TAG  # Do not destroy production\n      when: never\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_DESTROY == \"true\" # Manually destroy integration env.\n      when: manual\n###########################################################################################################\n\n\n###########################################################################################################\n\n## Dev env.\n\n##  * Temporary environment. Lives and dies with the Merge Request.\n\n##  * Auto-deploy on push to feature branch.\n\n##  * Auto-destroy on when Merge Request is closed.\n\n###########################################################################################################\n\nbuild_review:\n  extends: build\n  rules:\n    - if: $CI_COMMIT_TAG\n      when: never\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n      when: on_success\n\ndeploy_review:\n  extends: deploy\n  dependencies:\n    - build_review\n  environment:\n    name: $environment\n    action: start\n    on_stop: destroy_review\n    # url: https://$CI_ENVIRONMENT_SLUG.example.com\n  rules:\n    - if: $CI_COMMIT_TAG\n      when: never\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n      when: on_success\n\ndestroy_review:\n  extends: destroy\n  dependencies:\n    - build_review\n  environment:\n    name: $environment\n    action: stop\n  rules:\n    - if: $CI_COMMIT_TAG  # Do not destroy production\n      when: never\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH   # Do not destroy staging\n      when: never\n    - when: manual\n###########################################################################################################\n\n\n###########################################################################################################\n\n## Second layer\n\n##  * Deploys from main branch to qa env.\n\n##  * Deploys from tag to production.\n\n###########################################################################################################\n\n.2nd_layer:\n  stage: 2nd_layer\n  variables:\n    TF_ROOT: terraform\n  trigger:\n    include: .gitlab-ci/.second-layer.gitlab-ci.yml\n    # strategy: depend            # Do NOT wait for the downstream pipeline to finish to mark upstream pipeline as successful. Otherwise, all pipelines will fail when reaching the pipeline timeout before deployment to 2nd layer.\n    forward:\n      yaml_variables: true      # Forward variables defined in the trigger job\n      pipeline_variables: true  # Forward manual pipeline variables and scheduled pipeline variables\n\nqa:\n  extends: .2nd_layer\n  variables:\n    TF_STATE_NAME_2: qa\n    environment: $TF_STATE_NAME_2\n    TF_CLI_ARGS_plan_2: \"-var-file=../vars/$TF_STATE_NAME_2.tfvars\"\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n\nproduction:\n  extends: .2nd_layer\n  variables:\n    TF_STATE_NAME_2: production\n    environment: $TF_STATE_NAME_2\n    TF_CLI_ARGS_plan_2: \"-var-file=../vars/$TF_STATE_NAME_2.tfvars\"\n  rules:\n    - if: $CI_COMMIT_TAG\n###########################################################################################################\n\n```\n\n\nAt this stage, we are already deploying safely to three environments. That\nis my personal ideal recommendation. However, if you need more environments,\nadd them to your CD pipeline.\n\n\nYou have certainly already noted that we include a downstream pipeline with\nthe keyword `trigger:include`. This includes the file\n`.gitlab-ci/.second-layer.gitlab-ci.yml`. We want to run almost the same\npipeline so obviously, its content is very similar to the one we have\ndetailed above. The main advantage here to define this [grand-child\npipeline](#the-grand-child-pipeline) is that it lives on its own, making\nboth variables and rules way easier to define.\n\n\n### The grand-child pipeline\n\n\nThis second layer pipeline is a brand new pipeline. Hence, it needs to mimic\nthe first layer definition with:\n\n\n* [Inclusion of the Terraform\ntemplate](#run-terraform-commands-and-secure-the-code).\n\n* [Enforcement of security checks](#run-controls-on-all-branches). Terraform\nvalidation would be duplicates of the first layer, but security scanners may\nfind threats that did not yet exist when scanners previously ran (for\nexample, if you deploy to production a couple of days after your deployment\nto staging).\n\n* [Overwrite build and deploy jobs to set specific\nrules](#cd-to-review-environments). Note that the `destroy` stage is no\nlonger automated to prevent too fast deletions.\n\n\nAs explained above, the `TF_STATE_NAME` and `TF_CLI_ARGS_plan` have been\nprovided from the [main pipeline](#the-main-pipeline) to the [child\npipeline](#the-child-pipeline). We needed another variable name to pass\nthese values from the [child pipeline](#the-child-pipeline) to here, the\n[grand-child pipeline](#the-grand-child-pipeline). This is why they are\npostfixed with `_2` in the child pipeline and the value is copied back to\nthe appropriate variable during the `before_script` here.\n\n\nSince we have already broken down each step above, we can zoom out here\ndirectly to the broad view of the global second layer definition (stored in\n`.gitlab-ci/.second-layer.gitlab-ci.yml`).\n\n\n```yml\n\n# Use to deploy a second environment on both the default branch and the\ntags.\n\n\ninclude:\n  template: Terraform.gitlab-ci.yml\n\nstages:\n  - validate\n  - test\n  - build\n  - deploy\n\nfmt:\n  rules:\n    - when: never\n\nvalidate:\n  rules:\n    - when: never\n\nkics-iac-sast:\n  rules:\n    - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'\n      when: never\n    - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/\n      when: never\n    - when: always\n\n###########################################################################################################\n\n## QA env. and Prod. env\n\n##  * Manually trigger build and auto-deploy in QA\n\n##  * Manually trigger both build and deploy in Production\n\n##  * Destroy of these env. is not automated to prevent errors.\n\n###########################################################################################################\n\nbuild:  # terraform plan\n  cache:  # Use a shared cache or tagged runners to ensure terraform can run on apply and destroy\n    - key: $TF_STATE_NAME_2\n      fallback_keys:\n        - cache-$CI_DEFAULT_BRANCH\n      paths:\n        - .\n  environment:\n    name: $TF_STATE_NAME_2\n    action: prepare\n  before_script:  # Hack to set new variable values on the second layer, while still using the same variable names. Otherwise, due to variable precedence order, setting new value in the trigger job, does not cascade these new values to the downstream pipeline\n    - TF_STATE_NAME=$TF_STATE_NAME_2\n    - TF_CLI_ARGS_plan=$TF_CLI_ARGS_plan_2\n  rules:\n    - when: manual\n\ndeploy: # terraform apply\n  cache:  # Use a shared cache or tagged runners to ensure terraform can run on apply and destroy\n    - key: $TF_STATE_NAME_2\n      fallback_keys:\n        - cache-$CI_DEFAULT_BRANCH\n      paths:\n        - .\n  environment: \n    name: $TF_STATE_NAME_2\n    action: start\n  before_script:  # Hack to set new variable values on the second layer, while still using the same variable names. Otherwise, due to variable precedence order, setting new value in the trigger job, does not cascade these new values to the downstream pipeline\n    - TF_STATE_NAME=$TF_STATE_NAME_2\n    - TF_CLI_ARGS_plan=$TF_CLI_ARGS_plan_2\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n    - if: $CI_COMMIT_TAG && $TF_AUTO_DEPLOY == \"true\"\n    - if: $CI_COMMIT_TAG\n      when: manual\n###########################################################################################################\n\n```\n\n\nEt voilà. **We are ready to go.** Feel free to change the way you control\nyour job executions, leveraging for example GitLab's capacity to [delay a\njob](https://docs.gitlab.com/ee/ci/jobs/job_control.html#run-a-job-after-a-delay)\nbefore deploying to production.\n\n\n## Try it yourself\n\n\nWe finally reached our destination. We are now able to control **deployments\nto five different environments**, with only the **feature branches**, the\n**main branch**, and **tags**.\n\n* We are intensively reusing GitLab open source templates to ensure\nefficiency and security in our pipelines.\n\n* We are leveraging GitLab template capacities to overwrite only the blocks\nthat need custom control.\n\n* We have split the pipeline in small chunks, controlling the downstream\npipelines to match exactly what we need.\n\n\nFrom there, the floor is yours. You could, for example, easily update the\nmain pipeline to trigger downstream pipelines for your software source code,\nwith the\n[trigger:rules:changes](https://docs.gitlab.com/ee/ci/yaml/#ruleschanges)\nkeyword. And use another\n[template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/)\ndepending on the changes that happened. But that is another story.\n",[108,787,788,495,9],{"slug":4626,"featured":6,"template":700},"using-child-pipelines-to-continuously-deploy-to-five-environments","content:en-us:blog:using-child-pipelines-to-continuously-deploy-to-five-environments.yml","Using Child Pipelines To Continuously Deploy To Five Environments","en-us/blog/using-child-pipelines-to-continuously-deploy-to-five-environments.yml","en-us/blog/using-child-pipelines-to-continuously-deploy-to-five-environments",{"_path":4632,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4633,"content":4639,"config":4644,"_id":4646,"_type":13,"title":4647,"_source":15,"_file":4648,"_stem":4649,"_extension":18},"/en-us/blog/using-gitlab-web-ide-gitlab-ci-cd",{"title":4634,"description":4635,"ogTitle":4634,"ogDescription":4635,"noIndex":6,"ogImage":4636,"ogUrl":4637,"ogSiteName":685,"ogType":686,"canonicalUrls":4637,"schema":4638},"How to make small changes using GitLab’s Web IDE","A quick three minute demo shows how teams can deliver better apps faster using GitLab CI/CD.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749678812/Blog/Hero%20Images/web-ide-cover.jpg","https://about.gitlab.com/blog/using-gitlab-web-ide-gitlab-ci-cd","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"How to make small changes using GitLab’s Web IDE\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Chrissie Buchanan\"}],\n        \"datePublished\": \"2020-05-28\",\n      }",{"title":4634,"description":4635,"authors":4640,"heroImage":4636,"date":4641,"body":4642,"category":3780,"tags":4643},[3224],"2020-05-28","\n\nIt’s not enough to say something is quick and easy. To have a better understanding of some of the benefits of using [GitLab CI/CD](/topics/ci-cd/), it’s much better to _show_ you.\n\nIn a [short video](https://www.youtube.com/watch?v=6207TKNGgJs&feature=emb_logo), [Itzik Gan-Baruch](/company/team/#iganbaruch) technical marketing manager, demonstrates how to submit a code change using GitLab Web IDE. In three minutes, teams can submit a code change and commit it, trigger a CI pipeline to scan for any errors, and ship the updated application to users.\n\n## Getting started with GitLab Web IDE\n\nAll code that gets automatically tested and deployed to production has a human at its source. In GitLab 10.7, we released the [first iteration of our Web Integrated Development Environment (IDE)](/blog/introducing-gitlab-s-integrated-development-environment/) after observing how non-developers struggled with editing multiple files and committing those changes. Since we believe that [everyone can contribute](/company/mission/#mission), building an editor that was integrated with GitLab that made it easier for anyone to contribute seemed like a natural fit. To access the Web IDE, just click the button from any GitLab project.\n\n![Web IDE](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_1.png){: .shadow.medium.center}\n\nThe Web IDE button\n{: .note.text-center}\n\nIn this simple project with a job application, you can use the Web IDE to make a code change and push it to a feature branch. Select the file you would like to change from the menu on the left.\n\n![Selecting a file](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_2.png){: .shadow.medium.center}\n\nSelecting a file from the Wed IDE\n{: .note.text-center}\n\nOnce you’ve modified the text in that file, add a commit message and create a new branch. Click `Commit` to create a merge request.\n\n![Commit](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_3.png){: .shadow.medium.center}\n\nCommit to create a merge request\n{: .note.text-center}\n\nYour commit generates a merge request, and from here you can add an assignee, tie this code change to a specific milestone, add labels, or add any additional information regarding the change.\n\n![Modify merge request](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_4.png){: .shadow.medium.center}\n\nSubmit merge request\n{: .note.text-center}\n\nA new [continuous integration pipeline](/solutions/continuous-integration/) is triggered automatically. Click on the pipeline to see the stages.\n\n![Pipeline](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_5.png){: .shadow.medium.center}\n\nClick on the pipeline from the merge request\n{: .note.text-center}\n\nIn this project, the pipeline needed zero-configuration because it was generated through GitLab's [Auto DevOps](/direction/delivery/auto_devops/) capability. The pipeline has stages and a few jobs within each stage.\n\n![Auto DevOps pipeline](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_6.png){: .shadow.medium.center}\n\nA CI pipeline automatically configured with GitLab Auto DevOps\n{: .note.text-center}\n\nFirst, it builds a Docker image for the code and pushes it to the container registry. From there, it begins tests and scans jobs that run in parallel to help speed up the pipeline.\n\n![Pipeline jobs](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_7.png){: .shadow.medium.center}\n\nClick on a job within the pipeline stage to get more information\n{: .note.text-center}\n\nBy clicking on a job within the stage, you can see what happens.\n\n![dependency scan](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_8.png){: .shadow.medium.center}\n\nDependency scanning details\n{: .note.text-center}\n\nOnce all tests are completed, all test results will be added to the merge request that was created. The merge request is really the key to using GitLab as a code collaboration and [version control platform](/topics/version-control/). It’s simply a request to merge one branch into another.\n\n![merge requests](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_9.png){: .shadow.medium.center}\n\nMerge requests for this project\n{: .note.text-center}\n\n[Review Apps](https://docs.gitlab.com/ee/ci/review_apps/) are a way to visualize the changes that were made. Click `View app` once the pipeline has completed to access the staging environment.\n\n![Review apps](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_10.png){: .shadow.medium.center}\n\nSelect `View app` to access a staging environment once a pipeline completes.\n{: .note.text-center}\n\nIn this environment, only changes that were made in the merge request will be displayed. This link can be sent to others so they can view the changes from a web browser.\n\n![staging environment](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_12.png){: .shadow.medium.center}\n\nThe Review App for this project\n{: .note.text-center}\n\nFrom the merge request, you can see the test results, including changes to code quality and the security scans. This scan detected 20 new vulnerabilities. If you’d like more information, just click `Expand` on the right.\n\n![pipeline test results](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_13.png){: .shadow.medium.center}\n\nPipeline test results\n{: .note.text-center}\n\nOnce the results have been expanded, you can click on each one to get more details.\n\n![SAST scan](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_14.png){: .shadow.medium.center}\n\nSAST vulnerabilities detected\n{: .note.text-center}\n\nBy clicking on one of these results, you can see the file that caused the vulnerability as well as the problematic lines of code.\n\n![security report](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_15.png){: .shadow.medium.center}\n\nSecurity report\n{: .note.text-center}\n\nFrom this menu, you can choose to dismiss the vulnerability or create an issue so that someone can fix it. Details from the test will be added to the issue automatically.\n\n![new issue](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_16.png){: .shadow.medium.center}\n\nA new issue created to investigate a vulnerability\n{: .note.text-center}\n\nFrom your original merge request, you can collaborate with others and have them take a look at the proposed changes.\n\n![collaborate on merge request](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_17.png){: .shadow.medium.center}\n\nTag someone in a merge request to have them see your changes\n{: .note.text-center}\n\nOnce you’ve gathered feedback and all pipelines have passed, click the `merge` button to trigger a new pipeline to deploy your application to production\n\n![Web IDE](https://about.gitlab.com/images/blogimages/CI_demo_blog_May_28/CI_demo_18.png){: .shadow.medium.center}\n\nClick `merge` to trigger a deployment pipeline\n{: .note.text-center}\n\nThis workflow shows how anyone can contribute code without using a command line. The Web IDE makes it easy for anyone to make changes without introducing additional risks or quality issues, all from the GitLab interface.\n\nTo see this three-minute demo in real-time, just watch the video below.\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube-nocookie.com/embed/6207TKNGgJs\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n",[108,956,9],{"slug":4645,"featured":6,"template":700},"using-gitlab-web-ide-gitlab-ci-cd","content:en-us:blog:using-gitlab-web-ide-gitlab-ci-cd.yml","Using Gitlab Web Ide Gitlab Ci Cd","en-us/blog/using-gitlab-web-ide-gitlab-ci-cd.yml","en-us/blog/using-gitlab-web-ide-gitlab-ci-cd",{"_path":4651,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4652,"content":4656,"config":4662,"_id":4664,"_type":13,"title":4665,"_source":15,"_file":4666,"_stem":4667,"_extension":18},"/en-us/blog/vibe-coding-with-gitlab-duo-agent-platform-issue-to-mr-flow",{"config":4653,"title":4654,"description":4655},{"noIndex":6},"Vibe coding with GitLab Duo Agent Platform: 'Issue to MR' Flow","Learn how to update your application in minutes with our newest agent Flow that takes developers from idea to code.",{"title":4657,"description":4655,"authors":4658,"heroImage":947,"date":4659,"body":4660,"category":718,"tags":4661},"Vibe coding with GitLab Duo Agent Platform: Issue to MR Flow",[715],"2025-09-03","[GitLab Duo Agent Platform](https://about.gitlab.com/gitlab-duo/agent-platform/) (available now in Beta) provides a framework for AI agents to interact with GitLab resources like issues and merge requests, enabling complex, multistep tasks from concept to completion. Agent Platform offers conversational ([agentic chat](https://about.gitlab.com/blog/gitlab-duo-chat-gets-agentic-ai-makeover/)) and automated ([agent Flows](https://about.gitlab.com/blog/gitlab-18-3-expanding-ai-orchestration-in-software-engineering/)) experiences to assist with code generation, modernization, security vulnerability resolution, and project analysis — all while providing enterprise-grade security and customizable controls.\n\n\"Issue to MR\" is an agent Flow that streamlines turning a well-scoped issue into a draft merge request (MR). The Flow analyzes an issue’s description and requirements, opens a draft MR linked to the issue, creates a development plan, and proposes an implementation — right from the GitLab UI.\n\n## The developer challenge\n\nProduct tweaks such as rearranging a UI layout, adjusting component sizing, or making a minor workflow change shouldn't require hours of setup work. Yet developers find themselves caught in a frustrating cycle: hunting through codebases to locate the right files, creating branches, piecing together scattered changes across multiple components, and navigating complex review processes. And this is all before they can even see if their solution works. Development overhead transforms what should be quick iterations into time-consuming tasks, slowing down feedback loops and making simple product improvements feel like major undertakings.\n\n## How to use the Issue to MR Flow to accelerate an application update\n\nYou first need to fulfill these prerequisites before using the Issue to MR Flow.\n\nPrerequisites:\n\n* An existing issue with clear requirements and acceptance criteria. This will help GitLab Duo Agent Platform better understand what you're trying to achieve and improve the quality of your output.  \n* Project access at Developer or higher permissions as the Flow will be making edits to your project's source code.  \n* GitLab Duo Agent Platform enabled for your group or project, with Flows allowed. For this, go to your project’s **Settings > General > GitLab Duo > Allow flow execution** toggle and enable it. GitLab is committed to helping provide guardrails, so agentic AI features require turning on these toggles to protect sensitive projects and ensure only the projects you want are accessible to GitLab Duo Agent Platform.\n\nOnce you have fulfilled all the prerequisites above, you can follow these steps to take advantage of the Issue to MR Flow:\n\n1. Create a project issue that describes what you’d like GitLab Duo Agent Platform to accomplish for you. Provide as much detail as possible in the issue description. If the issue already exists, open it by going to **Plan > Issues** and clicking on the issue describing the update you want. Keep the issue well-scoped and specific.\n\n2. Below the issue header, click on **Generate MR with Duo** to kick off the Flow.\n\n3. If you’d like to track the progress of the agents working on implementing your issue, go to **Automate > Agent sessions** to see the live session log as agents plan and propose changes.\n\n4. When the pipeline completes, a link to the MR appears in the issue’s activity. Open it to review the summary and file-level changes.\n\n5. If you’d like to validate the proposed updates by GitLab Duo Agent Platform locally, you can pull the branch on your laptop, build and run your app, and verify that the update behaves as expected. If needed, make edits in the MR and proceed with normal review.\n\n6. If you’re happy with all the proposed application updates, merge the MR to the main branch.\n\n## Why the Issue to MR Flow works well for application changes\n\nThe Issue to MR Flow proposes code changes and updates the MR directly, so you spend less time locating files and only have to evaluate and review the result. In addition, the MR is automatically linked to the originating issue, keeping context tight for reviewers and stakeholders. Finally, you can monitor the agent session to understand what’s happening at each step.\n\n## Benefits of GitLab Duo Agent Platform\n\nGitLab Duo Agent Platform is [an agentic orchestration layer](https://about.gitlab.com/blog/gitlab-duo-agent-platform-public-beta/) that brings **full project context**, including planning to coding, building, securing, deploying, and monitoring, so agents can help across the entire software development lifecycle (SDLC), not just code editing.\n\n* Unified data model: GitLab Duo Agents operate on GitLab’s unified SDLC data, enabling higher-quality decisions and collaboration across tasks — including non-coding ones.\n\n* Security and compliance are built-in: GitLab Duo Agents run within enterprise guardrails and are usable even in highly-regulated or offline/air-gapped environments.\n\n* Interoperability and extensibility: Orchestrate flows across vendors and tools; connect external data via [MCP](https://about.gitlab.com/topics/ai/model-context-protocol/)/A2A for richer context.\n\n* Scale collaboration: GitLab Duo Agents work in the GitLab UI and IDEs, enabling many-to-many human-agent collaboration.\n\n* Discoverable and shareable: Find and share agents and Flows in a centralized AI Catalog.\n\n## Try the Issue to MR Flow today\n\nFor application updates, like a modest UI adjustment, the Issue to MR Flow helps you move from a clear issue to a reviewable MR quickly, with progress you can monitor and changes you can validate and merge through your standard workflow. It keeps context, reduces handoffs, and lets your team focus on quality rather than busywork.\n\nWatch the Issue to MR Flow in action:\n\n\u003C!-- blank line -->\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/BrrMHN4gXF4?si=J7beTgWOLxvS4hOw\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\u003C!-- blank line -->\n\n> Try the Issue to MR Flow on GitLab Duo Agent Platform now with a [free trial of GitLab Ultimate with Duo Enterprise](https://about.gitlab.com/free-trial/).\n\n\n",[720,694,696,9],{"featured":90,"template":700,"slug":4663},"vibe-coding-with-gitlab-duo-agent-platform-issue-to-mr-flow","content:en-us:blog:vibe-coding-with-gitlab-duo-agent-platform-issue-to-mr-flow.yml","Vibe Coding With Gitlab Duo Agent Platform Issue To Mr Flow","en-us/blog/vibe-coding-with-gitlab-duo-agent-platform-issue-to-mr-flow.yml","en-us/blog/vibe-coding-with-gitlab-duo-agent-platform-issue-to-mr-flow",{"_path":4669,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4670,"content":4675,"config":4680,"_id":4682,"_type":13,"title":4683,"_source":15,"_file":4684,"_stem":4685,"_extension":18},"/en-us/blog/vscode-extension-development-with-gitlab",{"title":4671,"description":4672,"ogTitle":4671,"ogDescription":4672,"noIndex":6,"ogImage":3447,"ogUrl":4673,"ogSiteName":685,"ogType":686,"canonicalUrls":4673,"schema":4674},"VS Code extension development with GitLab","As VS Code editor increases in popularity, find out how GitLab + VS Code can be used for extension development and how we develop the official GitLab VS Code extension.","https://about.gitlab.com/blog/vscode-extension-development-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"VS Code extension development with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tomas Vik\"}],\n        \"datePublished\": \"2020-11-30\",\n      }",{"title":4671,"description":4672,"authors":4676,"heroImage":3447,"date":4677,"body":4678,"category":785,"tags":4679},[3452],"2020-11-30","\n## What is Visual Studio Code (VSC)?\n\nMicrosoft Visual Studio Code (VS Code) is an extensible text editor. It's implemented in TypeScript and runs on Node 12 and Electron. It was [first released in 2015](https://github.com/microsoft/vscode/releases/tag/0.10.1), and since then, become widely popular[^2]. This post explains the basics about the development of VS Code extensions, shows how you can use GitLab for extension development, and shares how we build the official [GitLab VS Code extension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow).\n\n## VS Code editor key features\n\nFor me, the key feature of the VS Code editor is that it created a platform for extensions. That means not just providing an API for extensions (which editors have done since the '90s [^3]) but also providing a marketplace and seamless way of publishing and updating extensions.\n\nThere is also a fully open source version of the VS Code called [VSCodium](https://vscodium.com/). This version removes some proprietary Microsoft code from the distribution and is analogous to the Google Chrome and Chromium projects.\n\n## VS Code extension\n\nVS Code extension is a JavaScript or TypeScript app that runs in node and has access to the [VS Code Extension API](https://code.visualstudio.com/api). The convenient thing about this architecture is that the extension is like any other node app and has full access to the host machine and network. It can choose its own library for network connection, manipulating file systems, and also for rendering web UI.\n\n## Extension API\n\nThe extension API is implemented in TypeScript; it allows users to manipulate almost every aspect of the editor. After months of using it, I find the design elegant (with the exception of testing, which seems to be an afterthought in many areas of the API).\n\nThe main features of the API are manipulating and searching the files, editing text, creating custom left panels and status bars, debuggers, custom webview tabs, (Jupyter) notebook providers, and more. The API also provides a simple way to communicate with the user via input fields and quick-pick panels, as well as showing output with info, warning, or error messages.\n\n## Extension Marketplace\n\nIf you are familiar with either AppStore or PlayStore, you'll find VS Code has an equivalent store called [Visual Studio Marketplace](https://marketplace.visualstudio.com/search?target=VSCode), and unlike on its older siblings, everything[^4] is for free. Both the easy browsing experience for the user and the ease of use for a developer are differentiators for VS Code.\n\nAs a developer, you set up your [Azure Cloud token](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token) and then run `vsce publish` in your extension folder. That's it, within a few minutes, most of your users[^5] are running the latest and greatest version of your extension. This process greatly reduces the pressure on developers to get everything right before releasing, enabling faster iteration.\n\nThere is also an independent marketplace called [open-vsx](https://open-vsx.org/) used mainly by VSCodium but also by [GitPod](https://docs.gitlab.com/ee/integration/gitpod.html) and others.\n\n## Developing extensions in GitLab\n\nIf you'd like to try and develop your own extension, you can fork the [`gitlab-example-extension`](https://gitlab.com/viktomas/gitlab-example-extension) project. It contains a complete setup for linting, unit and integration testing, and publishing the extension to both [Visual Studio Marketplace](https://marketplace.visualstudio.com/search?target=VSCode) and [open-vsx](https://open-vsx.org/). Thanks to GitLab being a single platform for the whole [DevOps lifecycle](/topics/devops/), you can just push your code changes to GitLab, and CI/CD takes care of everything else. As always, if you find any useful tweaks, please submit an MR because [everyone can contribute](/company/mission/#mission).\n\nYou can see what the VS Code extension API offers in the [official documentation](https://code.visualstudio.com/api). You can then have a look at [extension examples](https://code.visualstudio.com/api/extension-guides/overview) and extend them to make the VS Code editor do almost anything you want.\n\n## Our extension: GitLab Workflow\n\nIn June the [GitLab Workflow extension became officially supported by GitLab](/blog/use-gitlab-with-vscode/). Since then we've done a lot of cleanup work and bug fixes. Recently, we released our first larger feature: [Inserting GitLab project snippets](https://about.gitlab.com/releases/2020/11/22/gitlab-13-6-released/#insert-gitlab-snippets-directly-in-vs-code).\n\nThe primary purpose of the extension is to integrate GitLab features into the editor, so users don't have to leave the editor to perform basic tasks such as read an issue description or create a snippet from the code. The extension is trying to plug in the GitLab features into an existing VS Code Extension API to both minimise the need for custom code and to make the experience as VS Code-like as possible.\n\nThere are several main areas of the VS Code Extension API that we take advantage of:\n\n### Commands\n\n[Commands](https://code.visualstudio.com/api/extension-guides/command) are a versatile concept for triggering actions. The most common way to trigger commands is to use the \u003Ckbd>Cmd\u003C/kbd>+\u003Ckbd>Shift\u003C/kbd>+\u003Ckbd>P\u003C/kbd> Command Palette. But commands can also be triggered from context menus, clicks on buttons, or even programmatically by other code in the extension. The most common example of triggering commands programatically is to call the `vscode.open` command with a URL as a parameter. GitLab workflow does that every time we open the GitLab web page[^6].\n\n![Command Palette](https://about.gitlab.com/images/blogimages/vscode-extension-development-with-gitlab/commands.png){: .shadow.medium.center}\nCommand Palette in GitLab Workflow\n{: .note .text-center}\n\n### Tree View\n\nVS Code uses the [Tree View](https://code.visualstudio.com/api/extension-guides/tree-view) for displaying the left panel. The panel shows the file tree for the project, changed Git files, an outline of the open file, full-text search results, and more. We use this Tree View panel to show lists of issues and merge requests.\n\n![Tree View](https://about.gitlab.com/images/blogimages/vscode-extension-development-with-gitlab/tree-view.png){: .shadow.medium.center}\nTree View in GitLab Workflow\n{: .note .text-center}\n\n### Status bar\n\n[Status bar](https://code.visualstudio.com/api/extension-capabilities/extending-workbench#status-bar-item) is the slim panel at the bottom of the editor. Any extension can add items to it. Extensions such as Git, spell checks, linters, and formatters all add items to the status bar to provide the user with quick feedback.\n\nThe GitLab Workflow extension shows the MR, issue, and pipeline for the current branch. It, for example, allows you to see if your pipeline failed after the last push.\n\n![Status bar](https://about.gitlab.com/images/blogimages/vscode-extension-development-with-gitlab/status-bar.png){: .shadow.medium.center}\nStatus bar in GitLab Workflow\n{: .note .text-center}\n\nAltogether the VS Code API provides a great foundation for bringing GitLab features closer to the editor. The GitLab VS Code extension is an exciting project that **you too can contribute to**. The best place to start is the [GitLab project page](https://gitlab.com/gitlab-org/gitlab-vscode-extension).\n\n[^2]: [17th most popular project on GitHub](https://github.com/search?p=2&q=stars%3A%3E100&s=stars&type=Repositories) at the time of writing (2020-11-20)\n[^3]: GNU Emacs supported Lisp extensions in [1985](https://en.wikipedia.org/wiki/Emacs#GNU_Emacs)\n[^4]: I haven't been able to find a paid extension in the store.\n[^5]: The auto-update feature is on by default in VS Code, but it can be turned off in which case your users are not going to auto-update.\n[^6]: [Using `vscode.open` in the GitLab Workflow](https://gitlab.com/search?utf8=%E2%9C%93&search=vscode.open&group_id=9970&project_id=5261717&scope=&search_code=true&snippets=false&repository_ref=main&nav_source=navbar)\n\n[Cover image](https://art.ljubicapetkovic.com/cc-licensed/) by [Ljubica Petkovic](https://art.ljubicapetkovic.com), licensed under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\n{: .note}\n",[232,9,108],{"slug":4681,"featured":6,"template":700},"vscode-extension-development-with-gitlab","content:en-us:blog:vscode-extension-development-with-gitlab.yml","Vscode Extension Development With Gitlab","en-us/blog/vscode-extension-development-with-gitlab.yml","en-us/blog/vscode-extension-development-with-gitlab",{"_path":4687,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4688,"content":4693,"config":4698,"_id":4700,"_type":13,"title":4701,"_source":15,"_file":4702,"_stem":4703,"_extension":18},"/en-us/blog/vscode-workflow-new-features",{"title":4689,"description":4690,"ogTitle":4689,"ogDescription":4690,"noIndex":6,"ogImage":3447,"ogUrl":4691,"ogSiteName":685,"ogType":686,"canonicalUrls":4691,"schema":4692},"Four new tools for your Visual Studio Code and GitLab tool belt","Learn about new features that can help you review MRs and interact with GitLab","https://about.gitlab.com/blog/vscode-workflow-new-features","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Four new tools for your Visual Studio Code and GitLab tool belt\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tomas Vik\"}],\n        \"datePublished\": \"2021-11-17\",\n      }",{"title":4689,"description":4690,"authors":4694,"heroImage":3447,"date":4695,"body":4696,"category":785,"tags":4697},[3452],"2021-11-17","\n\nIn our [previous post](/blog/mr-reviews-with-vs-code/), we talked about merge request (MR) Reviews. We explained how the GitLab Workflow extension helps you review MRs without leaving VS Code. Since releasing and polishing the MR reviews, we've been working on improvements to the extension. In this post, we will show you how the latest features fit into your workflow.\n\n### Do you have a lot to say? Use a snippet patch!\n\nOn GitLab's web UI there's the \"suggestions\" feature. It's handy for suggesting small changes in the MR review. The VS Code platform doesn't let us recreate the same experience, but the extension offers an alternative: Snippet patches.\n\nSnippet patches are code changes (git patches) of arbitrary size shared as GitLab snippets. Because they don't have a size limit, they are perfect for suggesting changes to multiple files during the MR review.\n\nThe extension has two commands, `Create snippet patch` and `Apply snippet patch`. These commands use `git diff` and `git apply`, respectively, which means people can still apply the snippet patch even if they don't use the GitLab Workflow extension.\n\nIf a suggestion in the comment is a hammer, then a snippet patch is a pneumatic tamping machine. Next time you'll review an MR, and you see a lot of space for improvement, remember the adage: \"A patch is worth a thousand words\".\n\n\u003Ciframe width=\"560\" height=\"315\" src=\"https://www.youtube-nocookie.com/embed/QQxpLoKJULQ\" title=\"YouTube video player\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen>\u003C/iframe>\n\n### What's going on with my pipeline? - Improved CI status display\n\nThe extension always showed the latest CI pipeline status in both the status bar and the sidebar. However, if you tried to gauge your pipeline status, you probably run into one or more surprises. The status was hard to understand. Sometimes it related to a different branch, or it was out of date.\n\nWe've made the pipeline status much more reliable and readable. For starters, you can now see individual jobs and their status in the sidebar. Click on any job, and the extension opens a browser window with the GitLab job page.\n\nWe also improved the consistency of showing the pipeline status. The status bar and sidebar are now in sync and always showing pipeline for the current branch.\n\nWe are excited about the cleaner code. It makes it easier for anyone to contribute functionality. If you'd be interested in giving it a shot, we recommend starting with the [Download artifacts from the latest pipeline](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/106) feature request. \n\n\n![VS Code status bar](https://about.gitlab.com/images/blogimages/2021-11-05-vscode-workflow-new-features/ci-pipeline-panel.png){: .shadow.medium.center}\nVS Code CI Pipeline status overview from GitLab extension.\n{: .note .text-center}\n\n### Make the MR your own - Working with checked out code\n\nTwo recent improvements play well together to make your review more interactive. They help you spend less time on actions that don't directly relate to reviewing code. These improvements let you check out the MR branch and open a local file during a review.\n\n#### Check out the MR branch\n\nYou can checkout any MR locally, as long as it is not coming from a forked project. Right-click the MR in the side tree and select \"Checkout MR Branch\". After the command finishes, you'll have the MR branch checked out in your project. Now you can review and run the code.\n\n\u003Cfigure class=\"video_container\">\n  \u003Cvideo src=\"https://gitlab.com/gitlab-org/gitlab-vscode-extension/uploads/db804234ed4d338dea31a27778dba72e/checkout-mr-branch.mp4\" controls=\"true\" data-setup=\"{}\" data-title=\"checkout-mr-branch\" preload=\"metadata\" width=\"560\">\u003C/video>\n\u003C/figure>\n{: .shadow.medium.center}\n\n#### Open a local file during a review\n\nWhen you look at a changed file in an MR, you can click on a small \"file\" icon in the top-right corner. The extension will open the same file in your local repository.\n\nIf your local branch is different from the MR branch, the local file might not be the same as the MR file.\n\nOpening the local file is useful when you want to explore the surroundings of the file quickly. The VS Code automatically focuses the file in the file tree, which lets you see all the neighbouring files.\n\n\u003Cfigure class=\"video_container\">\n  \u003Cvideo src=\"https://gitlab.com/gitlab-org/gitlab-vscode-extension/uploads/de2839b1ceb1be6c33cd80d7fe72bc6d/open-mr-file.mp4\" controls=\"true\" data-setup=\"{}\" data-title=\"open-mr-file\" preload=\"metadata\" width=\"560\">\u003C/video>\n\u003C/figure>\n{: .shadow.medium.center}\n\n### Commitment problems? Browse repositories without checking them out\n\nAt GitLab, we've got some large repositories. The largest, which all GitLabbers use daily, is [www-gitlab-com](https://gitlab.com/gitlab-com/www-gitlab-com), the website you see when you visit `about.gitlab.com`. This 6 GB colossus takes several minutes to check out.\n\nExploring this repository is a perfect use case for our latest feature: Remote Repositories, [contributed by Ethan Reesor](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/321), a community member.\n\nRun the `GitLab: Open Remote Repository` command, pick which project and branch you want to use, and _voilà_.  The extension opens the repository in your local workspace, but it doesn't store data on your local machine.\n\nRemote repositories are useful when you want to browse a repository for a reference but don't plan to change the code.\n\nThis is the first iteration, and it's got some limitations - you can't use full-text search, fuzzy file navigation, and the files are read-only. It's useful nonetheless.\n\n\u003Ciframe width=\"560\" height=\"315\" src=\"https://www.youtube-nocookie.com/embed/p4GTVx_Nd2s\" title=\"YouTube video player\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen>\u003C/iframe>\n\n### Thank you community!\n\nMost of the features introduced in this post are either implemented or suggested by a community member. Ahmed Mohamadeen [suggested](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/390) opening local file during MR review, Musisimaru [created initial implementation](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/203) of checking out MR branch, and Ethan Reesor [implemented](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/321) remote repositories.\n\n\nIf you'd like to shape the future of the GitLab Workflow VS Code extension, you can create issues in [our issue tracker](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues), or look for [issues where we accept MRs](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues?label_name%5B%5D=Accepting+merge+requests). Our [CONTRIBUTING](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/blob/main/CONTRIBUTING.md) guide is an excellent place to start.\n\nCover image by [Ljubica Petkovic](https://ljubicapetkovic.com), licensed under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\n{: .note}\n",[232,9,789],{"slug":4699,"featured":6,"template":700},"vscode-workflow-new-features","content:en-us:blog:vscode-workflow-new-features.yml","Vscode Workflow New Features","en-us/blog/vscode-workflow-new-features.yml","en-us/blog/vscode-workflow-new-features",{"_path":4705,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4706,"content":4711,"config":4716,"_id":4718,"_type":13,"title":4719,"_source":15,"_file":4720,"_stem":4721,"_extension":18},"/en-us/blog/vscode-workflows-for-working-with-gitlab",{"title":4707,"description":4708,"ogTitle":4707,"ogDescription":4708,"noIndex":6,"ogImage":3447,"ogUrl":4709,"ogSiteName":685,"ogType":686,"canonicalUrls":4709,"schema":4710},"Visual Studio code editor: Eight tips for using GitLab VS Code","Learn how to use the Visual Studio code editor more efficiently and meet some of the GitLab contributors that made these new features happen.","https://about.gitlab.com/blog/vscode-workflows-for-working-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Visual Studio code editor: Eight tips for using GitLab VS Code\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Tomas Vik\"}],\n        \"datePublished\": \"2021-05-20\",\n      }",{"title":4707,"description":4708,"authors":4712,"heroImage":3447,"date":4713,"body":4714,"category":785,"tags":4715},[3452],"2021-05-20","\n\nAs a software engineer, I spend a significant portion of my day in the Visual Studio code editor. Since I started maintaining the officially supported [GitLab VS Code extension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow), I've developed a few tricks that make me a productive GitLab user. Below, I share eight tips that make my work more efficient and productive, while also introducing you to some of the GitLab contributors who made this tooling happen.\n\n## What is Visual Studio Code?\n[Visual Studio Code](https://en.wikipedia.org/wiki/Visual_Studio_Code), developed by Microsoft, lets a user debug source code in various languages from the editor. It is also used for syntax highlighting, intelligent code completion, code refactoring, embedded Git and autocomplete. VS Code, as it is commonly known, can be launched or attached to running apps.\n\nIt is designed for Windows, Linux, and MacOS. VS Code can be used with several programming languages such as Java, JavaScript, Node.js, Python, C++ and Fortran. Support for additional languages is provided by freely available extensions on the [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow).\n\nBelow are eight tips for using GitLab VS Code.\n\n### How to clone any GitLab project\n\nGitLab contributor [Felix Haase](https://gitlab.com/haasef) recently [implemented a feature that lets you clone any GitLab project where you are a member](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/172). To clone the project, use the official `Git: Clone` command and select your GitLab instance. Use the `Git: Clone` command by selecting the command from the [Command Palette](https://code.visualstudio.com/docs/getstarted/userinterface#_command-palette).\n\nThis feature can save you time if you already know the name of the project you want to clone.\n\n![VS Code clone dialogue](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/clone.png){: .shadow.medium.center}\nVS Code lets you filter which project to clone.\n{: .note .text-center}\n\n### How to view MRs and issues\n\nIt is easy to look through issues and MRs that you created, are assigned to, or are reviewing using GitLab. The lesser-known feature of the GitLab Workflow extension is [custom queries](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/blob/main/docs/user/custom-queries.md). Custom search queries allow you to refine the search expressions for issues and MRs that appear in the VS Code side panel. You can apply all the advanced search terms you are used to from the GitLab web search: Labels, full-text search expression, milestones, authors, assignees, and more.\n\n![GitLab extension sidebar](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/issues-and-mrs.png){: .shadow.medium.center}\nSee your issues and MRs in the VS Code sidebar.\n{: .note .text-center}\n\nAnother option is [reviewing the MRs in VS Code](/blog/mr-reviews-with-vs-code/). The final functionality that is missing in MR review is [creating new comments on the MR diff](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/342), which we plan to ship by July 2021.\n\n### How to create an MR with two clicks\n\nIf you use the `git` command in a terminal, you might have noticed that pushing your branch to GitLab produces the following output:\n\n```txt\nremote: To create a merge request for my-new-branch, visit:\nremote: https://gitlab-instance.com/my-group/my-project/merge_requests/new?merge_request%5Bsource_branch%5D=my-new-branch\n```\n\nAfter clicking the link, the terminal will open your browser on a new MR page where you can create an MR from the branch you just pushed.\n\nWhen I started pushing my branches through VS Code, I missed this feature. To the point that I searched through the VS Code Git Extension logs to find the create MR link (command `Git: Show Git Output`).\n\nLuckily, GitLab contributor [Jonas Tobias Hopusch](https://gitlab.com/jotoho) implemented a [status bar button](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/155) that lets you create MRs just as easily.\n\nTo create an MR from your changes, push them to your remote repository (the cloud icon next to the branch name) and then click on the `GitLab: Create MR.` button.\n\n![VS Code status bar](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/status-bar-create-mr.png){: .shadow.medium.center}\nVS Code status bar with buttons from GitLab extension.\n{: .note .text-center}\n\n### How to configure your GitLab CI\n\nThe GitLab extension helps you edit your `.gitlab-ci.yml` configuration file in two ways: Autocompleting environment variables and validating the configuration.\n\nThanks to [Kev's](https://gitlab.com/KevSlashNull) fantastic [contribution](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/merge_requests/140), you can [use CI](/solutions/continuous-integration/) variable autocompletion anywhere in your `.gitlab-ci.yml`. The hints even include variable descriptions and explain supported GitLab versions.\n\n![CI variables autocomlete dialogue](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/ci-autocomplete.png){: .shadow.medium.center}\nCI variables autocomplete dialogue.\n{: .note .text-center}\n\nWhen you finish writing your `.gitlab-ci.yml` CI configuration, you can use the `GitLab: Validate GitLab CI config` command to surface any problems before committing the CI config to your repository.\n\n### How to create and paste project snippets\n\nIs there a piece of text that you and your teammates often use? Maybe it is a license header for a file or a test scenario template. You can use GitLab snippets in combination with Visual studio code editor to save you a few keystrokes.\n\nFor example, you can create a [test file snippet](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/snippets/2110322) with the `GitLab: Create snippet` command and then paste it into every new test file you create with the `GitLab: Insert snippet` command.\n\n![Paste Snippet dialogue](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/paste-snippet.png){: .shadow.medium.center}\nPaste Snippet dialogue.\n{: .note .text-center}\n\nI mostly use snippets when I want to share a big blob of text. I select the text and then create the snippet with the `GitLab: Create snippet` command.\n\n### How to copy web URL for a project file\n\nMost of the [communication at GitLab happens asynchronously](https://handbook.gitlab.com/handbook/values/#bias-towards-asynchronous-communication). So instead of being able to show your colleague an open file in your editor, you'll need to be able to create a textual pointer to the file.\n\nA straightforward way to do that is to use the `GitLab: Copy link to active file on GitLab` command, which will copy the web URL of the open file into your clipboard. It even includes the line number or a range of lines based on your cursor or selection in the Visual studio code editor.\n\nYou might also consider using the `GitLens: Copy Remote File URL`, which even includes the commit SHA in the URL, making it a permalink. The permalink will always point to the same version of the file regardless of further commits to your branch. We'll look at the GitLens extension in tip number 7 a bit later on.\n\nFor the GitLab Enterprise/Community Edition you can use:\n\"gitweblinks.gitLabEnterprise\": [\n    {\n        \"http\": \"https://local-gitlab\",\n        \"ssh\": \"git@local-gitlab\"\n    }\n]\n\n#### What to do if VS Code source control is not working\n\nA `SourceControl` is the entity responsible for populating the [Source Control model](https://code.visualstudio.com/api/extension-guides/scm-provider) with resource states, instances of `SourceControlResourceState`. Resource states are organized in groups, instances of `SourceControlResourceGroup`.\n\nLinking to issues in source code is a normal part of the VS Code workflow, especially when there's some logic that's difficult to understand or when there's a //TODO comment that needs action. [Users report experiencing issues](https://stackoverflow.com/questions/60232215/visual-studio-code-source-control-not-showing-changes) with changes in the file not appearing to the source code, unless inputted manually.\n\nOne user offered a 7-step solution that worked for them. Another said that all they had to do was disable and then reenable the build in Git extension, which fixed it. Yet another said they went to their “code” folder where they keep all their repos, right-clicked on the folder containing the repo they wanted and opening that folder with VS code.\n\nAn often-used approach to look at issues is to pick one to work on, create a branch to work in, make some commits, then merge your changes back into the main or default branch with a pull request. You can do that from the new Issues view.\n\n#### GitLab Workflow extensions for VS Code\n\nThe [GitLab Workflow extension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow) integrates GitLab with VS Code. You can decrease context switching and do more day-to-day tasks in VS Code, such as:\n\n- [View issues](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#browse-issues-review-mrs).\n- Run [common commands](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#commands) from the Visual Studio Code [command palette](https://code.visualstudio.com/docs/getstarted/userinterface#_command-palette).\n- Create and [review](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#merge-request-reviews) merge requests directly from Visual Studio Code.\n- [Validate](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#validate-gitlab-ci-configuration) your GitLab CI configuration.\n- [View the status](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#information-about-your-branch-pipelines-mr-closing-issue) of your current pipeline.\n- [Create](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#create-snippet) and paste snippets to, and from, your editor.\n- [Browse repositories](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#browse-a-repository-without-cloning) without cloning them\n\nDownload the extension from the [Visual Studio Code Marketplace](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow). Then you can configure:\n\n- [Features to display or hide](https://gitlab.com/gitlab-org/gitlab-vscode-extension#extension-settings).\n- [Self-signed certificate](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#self-signed-certificates) information\n\nReport any issues, bugs, or feature requests in the [gitlab-vscode-extension issue queue](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues).\n\nThis extension supports GitLab Version 13.0 and later. To find your GitLab version, visit [help](https://gitlab.com/help).\n\nYou can also see pipeline status, open MR and closing issue links in the status bar. The pipeline status is updated automatically so you don’t need to open GitLab to see your pipeline status.\n\nWithin the marketplace you can also use the command palette to run the commands and create a GitLab personal access token (required) and assign it to the extension.\n\nYou can also set set the token in an environment variable and learn how to change the VS Code settings. There are instructions for several other in-depth features as well.\n\n## How GitLens simplifies working with VS Code editor\n\nUp until now, the tips were centered around the [GitLab Workflow extension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow), but there is a fantastic extension that's improving VS Code git integration regardless of where you host your repository: [GitLens](https://marketplace.visualstudio.com/items?itemName=eamodio.gitlens).\n\n### Walking file history\n\nGitLens makes it easy to browse the history of changes to the current file. Each versioned file will have three new editor icons, which provides quick access to all previous revisions of the file. The middle button seen in the image below provides series of actions on the current version (e.g., opening the commit in GitLab web).\n\n![GitLens history browsing buttons](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/gitlens-history.png){: .shadow.medium.center}\nGitLens history browsing buttons\n{: .note .text-center}\n\n### How to compare current HEAD against branch or tag\n\nOne of my habits was inspecting `git diff` between my feature branch and the main branch before creating an MR. More often than not, I forgot to write a test or remove some pesky `console.log()`.\n\nGitLens adds multiple sections to your [\"Source Control\" tab](https://code.visualstudio.com/docs/editor/versioncontrol#_scm-providers). For each branch, tag, and commit, click a \"Compare\" icon which will show you changes between your current HEAD and the reference. Seeing the local diff is great for previewing changes before pushing the new branch to the remote.\n\n![GitLens - compare with branch](https://about.gitlab.com/images/blogimages/vscode-workflows-for-working-with-gitlab/gitlens-compare.png){: .shadow.medium.center}\nHow to compare with a branch using GitLens.\n{: .note .text-center}\n\n## Everyone can contribute\n\nNew features and fixes to the GitLab Visual Studio Code editor extension are added every month. If you find any issues or have a feature request, please go to our [GitLab VSCode issues tracker](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues) and if your request isn't already present in the tracker, create one. Everyone can contribute to GitLab, and we welcome your ideas on how to improve our Visual Studio Code editor.\n\n## Read more on Visual Studio and GitLab:\n\n- [Four new tools for your Visual Studio Code and GitLab tool belt](/blog/vscode-workflow-new-features/)\n\n- [VS Code extension development with GitLab](/blog/vscode-extension-development-with-gitlab/)\n\n- [How to do GitLab merge request reviews in VS Code](/blog/mr-reviews-with-vs-code/)\n\n- [How we created a GitLab Workflow Extension for VS Code](/blog/use-gitlab-with-vscode/)\n\n",[232,9],{"slug":4717,"featured":6,"template":700},"vscode-workflows-for-working-with-gitlab","content:en-us:blog:vscode-workflows-for-working-with-gitlab.yml","Vscode Workflows For Working With Gitlab","en-us/blog/vscode-workflows-for-working-with-gitlab.yml","en-us/blog/vscode-workflows-for-working-with-gitlab",{"_path":4723,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4724,"content":4730,"config":4735,"_id":4737,"_type":13,"title":4738,"_source":15,"_file":4739,"_stem":4740,"_extension":18},"/en-us/blog/vulnerability-risk-prioritization-made-simple-with-gitlab",{"title":4725,"description":4726,"ogTitle":4725,"ogDescription":4726,"noIndex":6,"ogImage":4727,"ogUrl":4728,"ogSiteName":685,"ogType":686,"canonicalUrls":4728,"schema":4729},"Vulnerability risk prioritization made simple with GitLab","GitLab provides detailed vulnerability risk data to assess the potential impact of detected vulnerabilities. Learn how this enables teams to effectively prioritize remediation efforts.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674528/Blog/Hero%20Images/blog-image-template-1800x945__5_.png","https://about.gitlab.com/blog/vulnerability-risk-prioritization-made-simple-with-gitlab","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Vulnerability risk prioritization made simple with GitLab\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Fernando Diaz\"}],\n        \"datePublished\": \"2025-03-12\",\n      }",{"title":4725,"description":4726,"authors":4731,"heroImage":4727,"date":4732,"body":4733,"category":830,"tags":4734},[1444],"2025-03-12","Development and security teams are often overwhelmed by the number of\nvulnerabilities they need to remediate. Many organizations remediate [less\nthan 16%](https://arxiv.org/pdf/2302.14172) of their known vulnerabilities\nmonthly. Vulnerability management teams face a constant challenge: which\nsecurity flaws deserve immediate attention? Three key frameworks help answer\nthis question: Common Vulnerability Scoring System\n([CVSS](https://nvd.nist.gov/vuln-metrics/cvss)), Known Exploited\nVulnerabilities\n([KEV](https://www.cisa.gov/known-exploited-vulnerabilities-catalog)), and\nExploit Prediction Scoring System ([EPSS](https://www.first.org/epss/)). The\n[GitLab 17.9\nrelease](https://about.gitlab.com/releases/2025/02/20/gitlab-17-9-released/)\nadds support for these frameworks. In this article, you'll learn how to use\nthese frameworks within GitLab to efficiently prioritize risk across your\ndependency and container image vulnerabilities using this data. \n\n\n![Vulnerability risk assessment\ndata](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674763/Blog/Content%20Images/vulnerability_data.png)\n\n\n\u003Ccenter>\u003Ci>Vulnerability risk assessment data\u003C/i>\u003C/center>\n\n\n## CVSS\n\n\nCVSS provides a standardized method for rating the severity of security\nvulnerabilities. Scores range from 0 to 10, with higher values indicating\ngreater severity.\n\n\nCVSS evaluates vulnerabilities across three dimension groups:\n\n\n* Base metrics: intrinsic qualities that don't change over time (attack\ncomplexity, privileges   required)  \n\n* Temporal metrics: factors that evolve (exploit maturity, remediation\nlevel)  \n\n* Environmental metrics: organization-specific impact considerations\n\n\nCVSS offers a consistent severity baseline and common language for security\nteams. Its comprehensive scoring methodology considers multiple aspects of a\nvulnerability's technical impact.\n\n\n## KEV\n\n\nThe Cybersecurity and Infrastructure Security Agency (CISA) maintains the\nKEV catalog, which identifies vulnerabilities actively exploited in the\nwild.\n\n\nUnlike academic severity scores, KEV focuses on real-world threat\nintelligence. Each entry includes:\n\n\n* CVE identifier  \n\n* Vulnerability name  \n\n* Action required  \n\n* Due date for remediation (for federal agencies)\n\n\nKEV provides actionable intelligence based on observed threat actor\nbehavior. It cuts through scoring complexity with a binary signal: \"This\nvulnerability is being actively exploited right now.\"\n\n\n## EPSS\n\n\nThe EPSS uses machine learning to predict the likelihood a vulnerability\nwill be exploited in the next 30 days. Scores range from 0 to 1 (or\n0%-100%), representing probability.\n\n\nEPSS analyzes hundreds of factors, including:\n\n* Technical characteristics  \n\n* Social media mentions  \n\n* Exploit availability  \n\n* Vulnerability age\n\n\nEPSS brings risk-based prioritization to vulnerability management. Rather\nthan focusing solely on technical severity, it helps teams understand which\nvulnerabilities attackers are most likely to target.\n\n\n## Combining the frameworks for effective prioritization\n\n\nEach framework serves a unique purpose:\n\n\n* CVSS indicates how severe a vulnerability is technically.  \n\n* KEV indicates which vulnerabilities are actively being exploited. \n\n* EPSS indicates which vulnerabilities are likely to be exploited soon.\n\n\nAn effective prioritization strategy leverages all three:\n\n\n1. Start with KEV-listed vulnerabilities as immediate priorities.  \n\n2. Use EPSS to identify high-probability threats not yet on KEV.  \n\n3. Consider CVSS for understanding technical impact.\n\n\nBy combining these complementary frameworks, security teams can focus\nlimited resources on the vulnerabilities that pose the greatest actual risk\nto their organizations. You can get started with prioritizing\nvulnerabilities with GitLab by:\n\n\n1. Adding security scanners to your pipeline  \n\n2. Viewing vulnerability insights  \n\n3. Setting the vulnerability status based metrics\n\n\nWatch this video to learn more:\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/7-dWwoKfCHw?si=iC73JCRsxPUEWKf-\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\n### Adding security scanners to your pipeline\n\n\nGitLab provides built-in security scanning tools through its templates that\ncan be integrated directly into your CI/CD pipeline. GitLab offers several\nsecurity scanners that address different aspects of your application\nsecurity:\n\n\n* **[Static Application Security Testing\n(SAST)](https://docs.gitlab.com/user/application_security/sast/):** Analyzes\nyour source code for known vulnerabilities  \n\n* **[Dynamic Application Security Testing\n(DAST)](https://docs.gitlab.com/user/application_security/dast/):** Tests\nyour running application for vulnerabilities  \n\n* **[Dependency\nScanning](https://docs.gitlab.com/user/application_security/dependency_scanning/):**\nChecks your dependencies for known vulnerabilities  \n\n* **[Container\nScanning](https://docs.gitlab.com/user/application_security/container_scanning/):**\nIdentifies vulnerabilities in container images  \n\n* **[Secret\nDetection](https://docs.gitlab.com/user/application_security/secret_detection/):**\nFinds secrets and credentials accidentally committed to your repository  \n\n* **[Infrastructure as Code\nScanning](https://docs.gitlab.com/user/application_security/iac_scanning/):**\nDetects security issues in IaC files  \n\n* **[Coverage-guided\nFuzzing](https://docs.gitlab.com/user/application_security/coverage_fuzzing/):**\nSends random inputs to an instrumented version of your application in an\neffort to detect bugs  \n\n* **[Web API\nFuzzing](https://docs.gitlab.com/user/application_security/api_fuzzing/):**\nSets operation parameters to unexpected values in an effort to cause\nunexpected behavior and errors in the API backend\n\n\nTo add them to your pipeline, simply add the appropriate templates to\n`.gitlab-ci.yml` file. For example, adding SAST and Dependency Scanning to\nyour pipeline is as simple as:\n\n\n```yaml  \n\ninclude:  \n  - template: Security/SAST.gitlab-ci.yml  \n  - template: Security/Dependency-Scanning.gitlab-ci.yml\n\nstages:  \n  - test  \n```\n\n\nOnce you commit the above changes, security scanners will begin to run.\nThese scanners can be further configured to meet the needs of your\norganization. To learn more about our various scanners, see the [GitLab\napplication security\ndocumentation](https://docs.gitlab.com/user/application_security/).\n\n\n**Note:** EPSS and KEV metrics are only provided for\n[dependency](https://docs.gitlab.com/user/application_security/dependency_scanning/)\nand [container\nimage](https://docs.gitlab.com/user/application_security/container_scanning/)\nvulnerabilities.\n\n\n### Viewing vulnerability insights\n\n\nOnce a pipeline with your security scanners is run on the default branch,\nyou can access the vulnerability report. The vulnerability report provides a\nconsolidated view of all security vulnerabilities detected across your\nproject by GitLab's security scanners. You can access it from your project\nby going to the side-tab and selecting **Secure > Vulnerability Report**.\n\n\n![Vulnerability report grouped by\ntool](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674763/Blog/Content%20Images/vulnerability_report__1_.png)\n\n\n\u003Ccenter>\u003Ci>Vulnerability report grouped by tool\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nFrom the vulnerability report, select a vulnerability to see its insights\npage, which includes the severity, EPSS, KEV, and CVSS along with the\nfollowing:\n\n\n* description  \n\n* when it was detected  \n\n* current status  \n\n* available actions  \n\n* linked issues  \n\n* actions log  \n\n* filename and line number of the vulnerability (if available)\n\n\nThis data can be used to effectively triage, remediate, or mitigate the\nvulnerability.\n\n\n__Note:__ From the insights page, you can also leverage GitLab Duo’s AI\ncapabilities to\n[explain](https://docs.gitlab.com/user/application_security/vulnerabilities/#vulnerability-explanation)\nand\n[auto-resolve](https://docs.gitlab.com/user/application_security/vulnerabilities/#vulnerability-resolution)\na vulnerability.\n\n\n### Setting the vulnerability status-based metrics\n\n\nAfter examining the provided data, we can go ahead and change the status of\nour vulnerability by clicking the **Change status** button:  \n\n\n![Change vulnerability status from insights\npage](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674764/Blog/Content%20Images/change_status.png)\n\n\n\u003Ccenter>\u003Ci>Change vulnerability status from insights page\u003C/i>\u003C/center>\n\n\u003Cbr>\u003C/br>\n\n\nThen we'll see a popup that will allow you to change the status of a\nvulnerability:\n\n\n![Change vulnerability status\noption](https://res.cloudinary.com/about-gitlab-com/image/upload/v1749674763/Blog/Content%20Images/change_status_2.png)\n\n\n\u003Ccenter>\u003Ci>Change vulnerability status option\u003C/i>\u003C/center>\n\n\n\u003Cbr>\u003C/br>\n\n\nWhen you dismiss a vulnerability you can choose one of the following reasons\nand optionally provide a comment:\n\n\n* **Acceptable risk:** The vulnerability is known, and has not been\nremediated or mitigated, but is considered to be an acceptable business\nrisk.  \n\n* **False positive:** An error in reporting in which a test result\nincorrectly indicates the presence of a vulnerability in a system when the\nvulnerability is not present.  \n\n* **Mitigating control:** The vulnerability’s risk is mitigated by a\nmanagement, operational, or technical control (that is, safeguard or\ncountermeasure) employed by an organization that provides equivalent or\ncomparable protection for an information system.  \n\n* **Used in tests:** The finding is not a vulnerability because it is part\nof a test or is test data.  \n\n* **Not applicable:** The vulnerability is known, and has not been\nremediated or mitigated, but is considered to be in a part of the\napplication that will not be updated.\n\n\nAnd there you have it, quick and easy vulnerability risk prioritization with\nGitLab!\n\n\n> Get started today with [a free trial of GitLab\nUltimate](https://about.gitlab.com/pricing/ultimate/)!\n\n\n## Learn more\n\n\nTo learn more about GitLab security and governance features and how we can\nhelp enhance your security posture, check out the following resources:\n\n\n* [GitLab Risk Assessment\nData](https://docs.gitlab.com/user/application_security/vulnerabilities/risk_assessment_data/)   \n\n* [GitLab Security and Compliance\nSolutions](https://about.gitlab.com/solutions/security-compliance/)  \n\n* [GitLab Application Security\ndocumentation](https://docs.gitlab.com/ee/user/application_security/)  \n\n* [GitLab Risk Assessment Data\nepic](https://gitlab.com/groups/gitlab-org/-/epics/11544)\n",[9,830,851],{"slug":4736,"featured":90,"template":700},"vulnerability-risk-prioritization-made-simple-with-gitlab","content:en-us:blog:vulnerability-risk-prioritization-made-simple-with-gitlab.yml","Vulnerability Risk Prioritization Made Simple With Gitlab","en-us/blog/vulnerability-risk-prioritization-made-simple-with-gitlab.yml","en-us/blog/vulnerability-risk-prioritization-made-simple-with-gitlab",{"_path":4742,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4743,"content":4749,"config":4756,"_id":4758,"_type":13,"title":4759,"_source":15,"_file":4760,"_stem":4761,"_extension":18},"/en-us/blog/why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies",{"title":4744,"description":4745,"ogTitle":4744,"ogDescription":4745,"noIndex":6,"ogImage":4746,"ogUrl":4747,"ogSiteName":685,"ogType":686,"canonicalUrls":4747,"schema":4748},"GitLab moves from compliance pipelines to security policies","Learn about our decision to deprecate compliance pipelines and how to migrate to pipeline execution policies. The process is detailed in this tutorial.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098128/Blog/Hero%20Images/Blog/Hero%20Images/security-checklist_security-checklist.png_1750098128272.png","https://about.gitlab.com/blog/why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Why GitLab is deprecating compliance pipelines in favor of security policies\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Ian Khor\"}],\n        \"datePublished\": \"2024-10-01\",\n      }",{"title":4750,"description":4745,"authors":4751,"heroImage":4746,"date":4753,"body":4754,"category":830,"tags":4755},"Why GitLab is deprecating compliance pipelines in favor of security policies",[4752],"Ian Khor","2024-10-01","GitLab compliance pipelines ensure security- and compliance-related jobs in applicable projects are run in accordance with compliance frameworks. Similarly, scan execution policies assure GitLab security scans are run in pipelines in a compliant manner.\n\nWhat we’ve learned from users is that they’d like to capture benefits offered by each feature through a single, simpler solution. Users would like to combine the flexibility of [compliance pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration) with the simplicity and versatility of [security policies](https://docs.gitlab.com/ee/user/application_security/policies/).\n\nTo meet this request, we developed a new feature, [pipeline execution policies](https://docs.gitlab.com/ee/user/application_security/policies/pipeline_execution_policies.html), to help users enforce customized CI/CD jobs for all applicable projects. Pipeline execution policies perform a similar function to compliance pipelines, but with increased focus on compliance enforcement, flexibility, and a foundation to build and solve for more use cases in the future.\n\nTo reduce confusion, compliance pipelines have been [deprecated](https://docs.gitlab.com/ee/update/deprecations.html#compliance-pipelines) in 17.3 now that pipeline execution policies are available and, as part of the deprecation, we are providing a step-by-step workflow for migrating from compliance pipelines to pipeline execution policy type in 17.5.\n\nYou can follow along with the work we’re doing with the deprecation through this [epic](https://gitlab.com/groups/gitlab-org/-/epics/11275).\n\n![compliance pipelines - image 1](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098139/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750098139599.png)\n\n## Why are we deprecating compliance pipelines?\n\nTo understand the reason behind this change, we first need to understand the difference between the [compliance management](https://about.gitlab.com/direction/govern/compliance/compliance-management/) and [policy management](https://about.gitlab.com/direction/govern/security_policies/security_policy_management/) features in GitLab. With compliance management, we are focused on helping you understand your compliance posture, providing tools to report to auditors, and surfacing compliance risks in a way that helps you take action.\n\nWe are also focused on increasing compliance visibility of framework requirements, violations, and audit events throughout the entire DevSecOps lifecycle. Our compliance management offering also establishes a direct association between controls and automations configured through policies back into compliance requirements established through compliance frameworks.\n\nPolicy management works hand in hand with compliance programs, as well as supporting scalable security initiatives. Policies give organizations a central location to globally enforce security controls, compliance controls, and automate security and compliance workflows. Security policies will continue to address core use cases across the lifecycle, such as defining enforcement around CI/CD component usage, blocking risks related to dependency and package management, and automating vulnerability management workflows to address security and compliance controls.\n\nTherefore, to ensure we provide the greatest value for our security and compliance users, we are deprecating compliance pipelines and providing a migration path for users to security policies. Not only does this make it clear and simple to the user how and when to enforce jobs as part of a project pipeline, but it also makes the distinction between compliance management and policy management in GitLab clearer. Compliance management is focused on compliance visibility, and policy management is focused on compliance and security enforcement across your entire GitLab instance.\n\n## What is the timeline for the deprecation and removal of compliance pipelines?\n\nThe iteration plan below can be found in the [issue that details the work we are doing](https://gitlab.com/groups/gitlab-org/-/epics/11275) to deprecate and remove compliance pipelines:\n\n**[Deprecation announcement](https://docs.gitlab.com/ee/update/deprecations.html#compliance-pipelines)**\n* Compliance pipeline deprecation and removal was announced in 17.3\n\n**[Compliance pipelines maintenance mode](https://gitlab.com/groups/gitlab-org/-/epics/12324)**\n* Adding banners and migration workflow, and docs\n* Released in 17.5\n\n**[Deter new compliance pipelines](https://gitlab.com/groups/gitlab-org/-/epics/14150)**\n* Adding warning banners for new pipelines\n* Encourage users to try the pipeline execution policy instead\n* Scheduled to start work on this 17.6\n* Scheduled to be released 17.8\n\n**[Compliance pipelines removal](https://gitlab.com/groups/gitlab-org/-/epics/12325) (Remove compliance pipelines)**\n* Provide tools to trial the removal and validate any errors\n* Scheduled to start work on this 17.8\n* Scheduled to be released 19.0\n\nAs you can see, we will start with the deprecation of compliance pipelines and the introduction of pipeline execution policy in the 17.3 release. \n\nLeading up to the removal of compliance pipelines in the 19.0 release, we are including new ways to inform and warn users about the upcoming removal. We are providing warning banners on new pipelines, as well as a workflow that can be used to migrate compliance pipelines to pipeline execution policy.\n\nWe ‘ll remove compliance pipelines in the 19.0 release, but provide a reverse feature flag in the milestones leading up that will help users test the removal and understand any impact prior to the removal date.\n\n## How to migrate your compliance pipelines to pipeline execution policy?\n\nThere are two ways users can access the workflow for migrating compliance pipelines to pipeline execution policy.\n\n1. When creating a new compliance framework, there will now be a warning banner that allows users to start using pipeline execution policy type instead of compliance frameworks:\n\n![compliance pipelines - image 3](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098140/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750098139599.png)\n\n2. When editing an existing compliance framework, there will now be a warning banner that enables users to migrate their compliance pipelines to pipeline execution policy type – if they have a compliance pipeline configured.\n\n![compliance pipelines - image 2](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098140/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750098139601.png)\n\nSelecting either \"Create policy\" or \"Migrate pipeline to a policy\" in either workflow will bring users to the \"New policy\" creation page in the \"Security Policies\" section. This will allow users to create a new security policy instead of a compliance pipeline. Or, if you migrate an existing compliance pipeline, the new policy will be populated with the compliance pipeline YAML as the remote source for the policy. Also, the policy scope will be populated with the framework from which you are migrating. \n\nThe policy will target all projects with that label for enforcement and apply enforcement of jobs defined in your remote file, now the pipeline execution YAML. By default, the new policy will be configured with the “override” mode, which will override downstream projects' `.gitlab-ci.yml` with the configuration you have defined (similar to compliance pipelines).\n\n![compliance pipelines - image 4](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750098140/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750098139604.png)\n\nAlternatively, you may use the “Inject” mode, which introduces a new set of reserved stages to run security and compliance jobs in isolation in a tamper-proof manner, without disrupting the project pipeline, and without coordinating with project teams to define stage names in their pipeline config. \n\nWith this approach, be sure to remove the `include:project`, which is no longer needed for this mode. And, depending on your version, ensure job names are unique (required in GitLab 17.2 and 17.3). In GitLab 17.4, we introduced additional enhancements for [managing conflicts](https://docs.gitlab.com/ee/user/application_security/policies/pipeline_execution_policies.html#job-naming-best-practice) for additional flexibility.\n\n## Start your migration today\n\nWe want to ensure that all GitLab users who are using compliance pipelines are fully aware of the deprecation of compliance pipelines in 17.3 and its eventual removal by the 19.0 release as a breaking change. \n\nWe are asking users to start migrating their compliance pipelines to the pipeline execution policy type as soon as possible, before the removal of compliance pipelines in GitLab 19.0.\n\nIf there are any questions, please contact your customer service representative or GitLab support for any help.\n\n> Follow along with the compliance pipeline deprecation progress in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/11275).\n\n> Share feedback in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/491924) regarding any gaps are blockers for adopting pipeline execution policies.",[830,9,851,108],{"slug":4757,"featured":90,"template":700},"why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies","content:en-us:blog:why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies.yml","Why Gitlab Is Deprecating Compliance Pipelines In Favor Of Security Policies","en-us/blog/why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies.yml","en-us/blog/why-gitlab-is-deprecating-compliance-pipelines-in-favor-of-security-policies",{"_path":4763,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4764,"content":4770,"config":4775,"_id":4777,"_type":13,"title":4778,"_source":15,"_file":4779,"_stem":4780,"_extension":18},"/en-us/blog/write-terraform-plans-faster-with-gitlab-duo-code-suggestions",{"title":4765,"description":4766,"ogTitle":4765,"ogDescription":4766,"noIndex":6,"ogImage":4767,"ogUrl":4768,"ogSiteName":685,"ogType":686,"canonicalUrls":4768,"schema":4769},"Write Terraform plans faster with GitLab Duo Code Suggestions","Follow this tutorial to learn how to use AI-powered code creation to manage your infrastructure with Terraform.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1749679194/Blog/Hero%20Images/duo-blog-post.png","https://about.gitlab.com/blog/write-terraform-plans-faster-with-gitlab-duo-code-suggestions","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"Write Terraform plans faster with GitLab Duo Code Suggestions\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Abubakar Siddiq Ango\"}],\n        \"datePublished\": \"2024-01-24\",\n      }",{"title":4765,"description":4766,"authors":4771,"heroImage":4767,"date":4772,"body":4773,"category":718,"tags":4774},[1604],"2024-01-24","[Terraform](https://www.terraform.io/) is an industry-standard for infrastructure orchestration. It can, however, be daunting and time-consuming to learn how to create infrastructure resources, especially when using Terraform providers you are unfamiliar with. That is where GitLab Duo Code Suggestions and AI-powered code creation comes in.\n\n[GitLab Duo Code Suggestions](https://about.gitlab.com/solutions/code-suggestions/) accelerates your coding in up to [15 supported programming languages](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/index.html#supported-languages-in-ides), including infrastructure as code (IaC) using Terraform.\n\nBy combining IaC with Terraform, infrastructure teams can adopt new Terraform providers/modules quickly and move changes to production faster. The combination also reduces the onboarding time for new users of Terraform or developers who are new to a code base by maintaining the context of the code base and dependencies in its suggestions. Instead of spending hours reading through documentation, you only need to review the given suggestions and update as necessary.\n\nIn this post, you will learn how to set up GitLab Duo Code Suggestions for Terraform.\n\n## How to set up GitLab Duo Code Suggestions for Terraform\n\nFollow these 6 steps to get started:\n\n1. Install the GitLab extension for your IDE of choice (read about [supported IDE extensions](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/index.html#supported-editor-extensions).\n2. Authenticate the GitLab extension with GitLab.com or your GitLab self-hosted instance.\n3. Enable Code Suggestions in your Profile settings under the Preferences > Preferences menu. Search for Code Suggestions, toggle the checkbox, and click Save.\n4. Confirm that Code Suggestions is also enabled in your GitLab extensions settings in the IDE.\n5. If you are using Visual Studio Code and Neovim, third-party extension support is needed to use Terraform with Code Suggestions. For VS Code, you can install the [official Terraform extension](https://marketplace.visualstudio.com/items?itemName=HashiCorp.terraform) from HashiCorp.\n6. To test and apply your Terraform plans, [install Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform) in your terminal and any necessary provider SDKs.\n\n## Creating your plans\n\nOnce you have all the prerequisites in place, all you need to do next is create a Terraform file ending with `.tf` and prompt GitLab Duo with comments describing what you want. Suggestions will be displayed while maintaining context. Even when no prompt is provided, GitLab Duo will suggest other Terraform resources you might be interested in based on the current context in your plans.\n\nLet's see this in action with GitLab Duo helping with a Terraform plan for provisioning a load balancer with four instances and other necessary resources on the Google Cloud Platform.\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/saa2JJ57UaQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n## Get started with Code Suggestions and Terraform today\n\n[Try Code Suggestions](https://about.gitlab.com/solutions/code-suggestions/) today and accelerate your software development. Code Suggestions is also available for [self-managed GitLab](https://docs.gitlab.com/ee/user/project/repository/code_suggestions/self_managed.html) (needs to be enabled by an administrator). Code Suggestions uses best-in-class large language models, and non-public customer data is never used as training data.",[495,720,9],{"slug":4776,"featured":90,"template":700},"write-terraform-plans-faster-with-gitlab-duo-code-suggestions","content:en-us:blog:write-terraform-plans-faster-with-gitlab-duo-code-suggestions.yml","Write Terraform Plans Faster With Gitlab Duo Code Suggestions","en-us/blog/write-terraform-plans-faster-with-gitlab-duo-code-suggestions.yml","en-us/blog/write-terraform-plans-faster-with-gitlab-duo-code-suggestions",{"_path":4782,"_dir":245,"_draft":6,"_partial":6,"_locale":7,"seo":4783,"content":4789,"config":4794,"_id":4796,"_type":13,"title":4797,"_source":15,"_file":4798,"_stem":4799,"_extension":18},"/en-us/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat",{"title":4784,"description":4785,"ogTitle":4784,"ogDescription":4785,"noIndex":6,"ogImage":4786,"ogUrl":4787,"ogSiteName":685,"ogType":686,"canonicalUrls":4787,"schema":4788},"10 best practices for using AI-powered GitLab Duo Chat","Explore tips and tricks for integrating GitLab Duo Chat into your AI-powered DevSecOps workflows. Plus, expert advice on how to refine chat prompts for the best results.","https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097639/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%281%29_77JeTV9gAmbXM0224acirV_1750097638765.png","https://about.gitlab.com/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat","\n                        {\n        \"@context\": \"https://schema.org\",\n        \"@type\": \"Article\",\n        \"headline\": \"10 best practices for using AI-powered GitLab Duo Chat\",\n        \"author\": [{\"@type\":\"Person\",\"name\":\"Michael Friedrich\"}],\n        \"datePublished\": \"2024-04-02\",\n      }",{"title":4784,"description":4785,"authors":4790,"heroImage":4786,"date":4791,"body":4792,"category":718,"tags":4793},[1646],"2024-04-02","Getting into a conversation with AI can be challenging. What question do you\nstart with? How do you frame the question? How much context is needed? Will\nthe conversation provide the best and most efficient results?\n\n\nIn this tutorial, we explore 10 tips and best practices to integrate GitLab\nDuo Chat into your AI-powered DevSecOps workflows and refine your prompts\nfor the best results.\n\n\n[Get started: Keep GitLab Duo Chat open and in\nsight](#get-started-keep-gitlab-duo-chat-open-and-in-sight)\n\n\n[10 best practices for using GitLab Duo\nChat](#10-best-practices-for-using-gitlab-duo-chat)\n\n\n1. [Have a conversation](#1.-have-a-conversation)\n\n2. [Refine the prompt for more\nefficiency](#2.-refine-the-prompt-for-more-efficiency)\n\n3. [Follow prompt patterns](#3.-follow-prompt-patterns)\n\n4. [Use low-context communication](#4.-use-low-context-communication)\n\n5. [Repeat yourself](#5.-repeat-yourself)\n\n6. [Be patient](#6.-be-patient)\n\n7. [Reset and start anew](#7.-reset-and-start-anew)\n\n8. [Gain efficiency with slash commands in the\nIDE](#8.-gain-efficiency-with-slash-commands-in-the-ide)\n\n9. [Refine the prompt for slash\ncommands](#9.-refine-the-prompt-for-slash-commands)\n\n10. [Get creative with slash\ncommands](#10.-get-creative-with-slash-commands)\n\n\nBonus content:\n\n- [Shortcuts](#shortcuts)\n\n- [Fun exercises](#fun-exercises)\n\n- [Learn more](#learn-more)\n\n\n> Live demo! Discover the future of AI-driven software development with our\nGitLab 17 virtual launch event. [Register\ntoday!](https://about.gitlab.com/seventeen/)\n\n\n## Get started: Keep GitLab Duo Chat open and in sight\n\n\n[GitLab Duo Chat](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html) is\navailable in the GitLab UI, Web IDE, and supported programming IDEs, for\nexample, VS Code. \n\n\nIn VS Code, you can open GitLab Duo Chat in the default left pane. You can\nalso drag and drop the icon into the right pane. This allows you to keep\nChat open while you write code and navigate the file tree, perform Git\nactions, etc. To reset the Chat location, open the command palette (by\npressing the `Command+Shift+P` (on macOS) or `Ctrl+Shift+P` (on\nWindows/Linux) keyboard shortcut and then type `View: Reset View Locations`.\nThe following short video shows you how to do it.\n\n\n\u003C!-- blank line -->\n\n\u003Cfigure class=\"video_container\">\n  \u003Ciframe src=\"https://www.youtube.com/embed/foZpUvWPRJQ\" frameborder=\"0\" allowfullscreen=\"true\"> \u003C/iframe>\n\u003C/figure>\n\n\u003C!-- blank line -->\n\n\nThe Web IDE and VS Code share the same framework – the same method works in\nthe Web IDE for more efficient workflows.\n\n\n![Chat in Web\nIDE](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image1_aHR0cHM6_1750097645344.png)\n\n\n## 10 best practices for using GitLab Duo Chat\n\n\n### 1. Have a conversation\n\n\nChats are conversations, not search forms.\n\n\nFor the first conversation icebreaker, you can start with the same search\nterms similar to a browser search and experiment with the response and\noutput. In this example, let's start with a C# project and best practices. \n\n\n> c# start project best practices\n\n\n![Chat prompt for C# start project best practices and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097646/Blog/Content%20Images/Blog/Content%20Images/image11_aHR0cHM6_1750097645345.png)\n\n\nThe response is helpful to understand a broad scope of C#, but does not\nkickstart immediate best practices. Let's follow up with a more focused\nquestion in the same context. \n\n\n> Please show the project structure for the C# project.\n\n\n![Chat prompt for project structure for the C# project and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image9_aHR0cHM6_1750097645346.png)\n\n\nThis answer is helpful. Next, let's follow up with a Git question, and use\nthe same question structure: Direct request to show something.\n\n\n> Show an example for a .gitignore for C#\n\n\n![Chat prompt for a .gitignore for C# and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image25_aHR0cHM6_1750097645347.png)\n\n\nContinue with CI/CD and ask how to build the C# project.\n\n\n> Show a GitLab CI/CD configuration for building the C# project\n\n\n![Chat prompt for GitLab CI/CD configuration for building C# project and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image16_aHR0cHM6_1750097645349.png)\n\n\nIn this example, Chat encouraged us to request specific changes. Let's ask\nto use the .NET SDK 8.0 instead of 6.0. \n\n\n> In the above example, please use the .NET SDK 8.0 image\n\n\n![Chat prompt to use .NET SDK 8.0 image and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image32_aHR0cHM6_1750097645350.png)\n\n\nThe CI/CD configuration uses the .NET command line interface (CLI). Maybe we\ncan use that for more efficient commands to create the projects and tests\nstructure, too? \n\n\n> Explain how to create projects and test structure on the CLI \n\n\n![Chat prompt to explain how to create projects and test structure on the\nCLI and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image14_aHR0cHM6_1750097645351.png)\n\n\nOf course, we could execute these commands in the terminal, but what if we\nwanted to stay in VS Code? Let's ask Chat.\n\n\n> Explain how to open a new terminal in VS Code\n\n\n![Chat prompt to explain how to open a new terminal in VS Code and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image5_aHR0cHM6_1750097645351.png)\n\n\n### 2. Refine the prompt for more efficiency\n\n\nThink of GitLab Duo Chat as a human, and engage with full sentences that\nprovide as much context into your thoughts and questions. \n\n\nExperienced browser search users might know this approach to queries: Build\nup the question, add more terms to refine the scope, and restart the search\nafter opening plenty of tabs. \n\n\nIn a browser search, this probably would result in four to five different\nsearch windows. \n\n\n```markdown\n\nc# start project best practices\n\nc# .gitignore\n\nc# gitlab cicd \n\nc# gitlab security scanning \n\nc# solutions and projects, application and tests\n\n``` \n\n\nYou can follow this strategy in a chat conversation, too. It requires adding\nmore context, making it a conversational approach. GitLab Duo Chat enables\nyou to ask multiple questions in one conversation request. Example: You need\nto start with a new C# project, apply best practices, add a `.gitignore`\nfile, and configure CI/CD and security scanning, just like in the above\nsearch. In Chat, you can combine the questions into one request.\n\n\n> How can I get started creating an empty C# console application in VS Code?\nPlease show a .gitignore and .gitlab-ci.yml configuration with steps for C#,\nand add security scanning for GitLab. Explain how solutions and projects in\nC# work, and how to add a test project on the CLI.\n\n\n![Chat prompt adding more context and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image37_aHR0cHM6_1750097645352.png)\n\n\nIn this response, Chat suggests to ask for specific configuration examples\nin follow-up questions in the conversation. Async practice: Create follow-up\nquestions. You can omit `C#` as context in the same chat session.\n\n\n> Please show an example for a .gitignore. Please show a CI/CD\nconfiguration. Include the SAST template.\n\n\n### 3. Follow prompt patterns \n\n\nFollow the pattern: `Problem statement, ask for help, provide additional\nrequests`. Not everything comes to mind when asking the first question –\ndon't feel blocked, and instead start with `Problem statement, ask for help`\nin the first iteration. \n\n\n> I need to fulfill compliance requirements. How can I get started with\nCodeowners and approval rules?\n\n\n![Chat prompt to get started with Codeowners and approval rules and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image19_aHR0cHM6_1750097645352.png)\n\n\nThe answer is helpful but obviously generic. Now, you may want to get\nspecific help for your team setup. \n\n\n> Please show an example for Codeowners with different teams: backend,\nfrontend, release managers.\n\n\n![Chat prompt to show an example for Codeowners with different teams:\nbackend, frontend, release managers and reponse\n](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image31_aHR0cHM6_1750097645353.png)\n\n\nAn alternative is to describe the situation you are in and to ask for input.\nIt can feel a bit like a conversation to follow the STAR model (Situation,\nTask, Action, Results). \n\n\n> I have a Kubernetes cluster integrated in GitLab. Please generate a Yaml\nconfiguration for a Kubernetes service deployment. Explain how GitOps works\nas a second step. How to verify the results?\n\n\n![Chat prompt with multiple questions and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image27_aHR0cHM6_1750097645354.png)\n\n\n### 4. Use low-context communication \n\n\nProvide as much context as needed to provide an answer. Sometimes, the\nprevious history or opened source code does not provide that helpful\ncontext. To make questions more efficient, apply a pattern of [low-context\ncommunication](https://handbook.gitlab.com/handbook/company/culture/all-remote/effective-communication/#understanding-low-context-communication),\nwhich is used in all-remote communication at GitLab.\n\n\nThe following question did not provide enough context in a C++ project.\n\n\n> Should I use virtual override instead of just override?\n\n\n![Chat prompt asking if the users should use virtual override instead of\njust override and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image34_aHR0cHM6_1750097645354.png)\n\n\nInstead, try to add more context:\n\n\n> When implementing a pure virtual function in an inherited class, should I\nuse virtual function override, or just function override? Context is C++. \n\n\n![Chat prompt with more detail and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image36_aHR0cHM6_1750097645355.png)\n\n\nThe example is also shown in the [GitLab Duo Coffee Chat: Refactor C++\nfunctions into OOP classes for abstract database\nhandling](https://youtu.be/Z9EJh0J9358?t=2190). \n\n\n### 5. Repeat yourself\n\n\nAI is not predictable. Sometimes, it may not answer with the expected\nresults, or does not produce source code examples or configuration snippets\nbecause it lacked context. It is recommended to repeat the question and\nrefine the requirements.\n\n\nIn the following example, we want to create a C# application. In the first\nattempt, we did not specify the application type – C# can be used to create\nconsole/terminal but also UI applications. The result also does not provide\nan empty example source code. The second, repeated prompt adds two more\nwords - `console` and `empty`. \n\n\n> How can I get started creating an C# application in VSCode?\n\n> \n\n> How can I get started creating an empty C# console application in VSCode?\n\n\nThe results in the prompt differ. The first response is helpful to get\nstarted by following the instructions in the VS Code window, but it does not\ntell us where the source code is located and how to modify it. The repeated\nprompt with refinements modifies the response and provides instructions how\nto override the default template with some “hello world” code.\n\n\n![Chat prompt with repeated prompt with modifications and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image28_aHR0cHM6_1750097645355.png)\n\n\nYou can also combine repeat and refine strategies, and ask Chat to show an\nexample for application code and tests.\n\n\n> How can I get started creating an empty C# console application in VSCode?\nPlease show an example for application and tests.\n\n\n![Chat prompt that asks for example for application and tests and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image3_aHR0cHM6_1750097645356.png)\n\n\n#### Repeat yourself after generic questions \n\n\nWhen asking generic technology questions, GitLab Duo Chat might not be able\nto help. In the following scenario, I wanted to get a suggestion for Java\nbuild tools and framework, and it did not work. There could be many answers:\nMaven, Gradle, etc., as build tools, and [100+ Java\nframeworks](https://en.wikipedia.org/wiki/List_of_Java_frameworks),\ndepending on the technology stack and requirements.\n\n\n![Chat prompt for Java build tools and framework and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image2_aHR0cHM6_1750097645356.png)\n\n\nLet's assume that we want to focus on a customer environment with [Java\nSpring Boot](https://spring.io/projects/spring-boot). \n\n\n> I want to create a Java Spring Boot application. Please explain the\nproject structure and show a hello world example.\n\n\n![Chat prompt that asks for more, including a hello world example and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image26_aHR0cHM6_1750097645357.png)\n\n\nThis provides great results already. As an async exercise, repeat the\nprompt, and ask how to deploy the application, adding more refinements in\neach step. Alternatively, you can make it a follow-up conversation.\n\n\n> I want to create a Java Spring Boot application. Please explain the\nproject structure and show a hello world example. Show how to build and\ndeploy the application in CI/CD.\n\n> \n\n> I want to create a Java Spring Boot application. Please explain the\nproject structure and show a hello world example. Show how to build and\ndeploy the application in CI/CD, using container images.\n\n> \n\n> I want to create a Java Spring Boot application. Please explain the\nproject structure and show a hello world example. Show how to build and\ndeploy the application in CI/CD, using container images. Use Kubernetes and\nGitOps in GitLab.\n\n\n### 6. Be patient\n\n\nSingle words or short sentences might not generate the desired results, [as\nshown in this video example](https://youtu.be/JketELxLNEw?t=1220).\nSometimes, GitLab Duo Chat is able to guess from available data, but\nsometimes also might insist on providing more context.\n\n\nExample: `labels` matches the GitLab documentation content.\n\n\n![Chat prompt about labels and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image12_aHR0cHM6_1750097645357.png)\n\n\nRefine the question to problem statements and more refinements for issue\nboard usage.\n\n\n> Explain labels in GitLab. Provide an example for efficient usage with\nissue boards.\n\n\n![Chat prompt that includes asking for an example and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image21_aHR0cHM6_1750097645358.png)\n\n\nOr use a problem statement, followed by a question and the ask for\nadditional examples.\n\n\n> I don't know how to use labels in GitLab. Please provide examples, and how\nto use them for filters in different views. Explain these views with\nexamples.\n\n\n![Chat prompt with problem statement and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image10_aHR0cHM6_1750097645358.png)\n\n\nAlso, avoid `yes/no` questions and instead add specific context.\n\n\n> Can you help me fix performance regressions?\n\n\n![Chat promptt that asks for help with fixing performance regressions and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image18_aHR0cHM6_1750097645359.png)\n\n\nInstead, provide the context of the performance regression, including the\nprogramming languages, frameworks, technology stack, and environments. The\nfollowing example uses an environment from some years ago, which can still\nbe accurate today.\n\n\n> My PHP application encounters performance regressions using PHP 5.6 and\nMySQL 5.5. Please explain potential root causes, and how to address them.\nThe app is deployed on Linux VMs.\n\n\n![Chat prompt that includes more detail and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image24_aHR0cHM6_1750097645360.png)\n\n\n### 7. Reset and start anew\n\n\nSometimes, the chat history shows a different learning curve and provides\nthe wrong context for follow-up questions. Or, you asked specific questions\nwhere GitLab Duo Chat cannot provide answers. Since generative AI is not\npredictable, it might also lack the ability to provide certain examples, but\nthink it gave them in a future response (observed in Chat Beta). The\nunderlying large language models, or LLMs, sometimes might insist on giving\na specific response, in an endless loop.\n\n\n> How can I get started creating an empty C# console application in VSCode?\nPlease show a .gitignore and .gitlab-ci.yml configuration with steps for C#,\nand add security scanning for GitLab. Explain how solutions and projects in\nC# work, and how to add a test project on the CLI.\n\n\nAfter asking the question above with an example configuration, I wanted to\nreduce the scope of the question to get a more tailored response. It did not\nwork as expected, since Chat knows about the chat history in context, and\nrefers to previous answers.\n\n\n> How can I get started creating an empty C# console application in VSCode?\nPlease show a .gitignore and .gitlab-ci.yml configuration with steps for C#.\n\n\n![Chat prompt that asks for configuration examples and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image23_aHR0cHM6_1750097645360.png)\n\n\nTo force Chat into a new context, use `/reset` as slash command to reset the\nsession, and repeat the question to get better results. You can also use\n`/clean` or `/clear` to delete all messages in the conversation.\n\n\n### 8. Gain efficiency with slash commands in the IDE \n\n\n#### Explain code\n\n\n- Q: Generated code? Existing code? Legacy code?\n\n- A: Use the [`/explain` slash command in the\nIDE](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#explain-code-in-the-ide).\n\n- A2: Refine the prompt with more focused responses, for example: `/explain\nfocus on potential shortcomings or bugs`. \n\n\n![Chat prompt with /explain slash\ncommand](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/gitlab_duo_chat_slash_commands_explain_01_aHR0cHM6_1750097645361.png)\n\n\n![Chat prompt with refined\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image6_aHR0cHM6_1750097645361.png)\n\n\n#### Refactor code \n\n\n- Q: Unreadable code? Long spaghetti code? Zero test coverage?\n\n- A: Use the [`/refactor` slash command in the\nIDE](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#refactor-code-in-the-ide). \n\n- A2: Refine the prompt for more targeted actions, for example\nobject-oriented patterns: `/refactor into object-oriented classes with\nmethods and attributes`. \n\n\n![Chat prompt with /refactor slash\ncommand](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image35_aHR0cHM6_1750097645362.png)\n\n\n![Chat prompt with refined\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image30_aHR0cHM6_1750097645362.png)\n\n\n#### Generate tests\n\n\n- Q: Testable code but writing tests takes too much time?\n\n- A: Use the [`/tests` slash command in the\nIDE](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html#write-tests-in-the-ide).\n\n- A2: Refine the prompt for specific test frameworks, or test targets. You\ncan also instruct the prompt to focus on refactoring, and then generate\ntests: `/tests focus on refactoring the code into functions, and generate\ntests`.\n\n\n![Chat prompt with /tests slash\ncommand](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image29_aHR0cHM6_1750097645363.png)\n\n\n![Chat prompt with refined\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image4_aHR0cHM6_1750097645363.png)\n\n\nMore practical examples in complete development workflows are available in\nthe [GitLab Duo\nexamples](https://docs.gitlab.com/ee/user/gitlab_duo_examples.html)\ndocumentation.\n\n\n### 9. Refine the prompt for slash commands \n\n\nYou will see refined prompts tips in this blog post a lot. It is one of the\ningredients for better AI-powered workflow efficiency. Slash commands are no\ndifferent, and allow for better results in GitLab Duo Chat.\n\n\nA customer recently asked: \"Can code explanations using `/explain` create\ncomments in code?\" The answer is: no. But you can use the Chat prompt to ask\nfollow-up questions, and ask for a summary in a code comment format. It\nrequires the context of the language. \n\n\nThe following example with a [C++ HTTP client code using the curl\nlibrary](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts/-/blob/5cc9bdd65ee8ee16c548bea0402c18f8209d4d06/chat/slash-commands/c++/cli.cpp)\nneeds more documentation. You can refine the `/explain` prompt by giving\nmore refined instructions to explain the code by adding code comments, and\nthen copy-paste that into the editor.\n\n\n> /explain add documentation, rewrite the code snippet\n\n\n![Chat prompt to add documentation and rewrite code snippet and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image13_aHR0cHM6_1750097645363.png)\n\n\nAlternatively, you can ask Chat to `/refactor` the source code, and generate\nmissing code comments through a refined prompt.\n\n\n> /refactor add code comments and documentation\n\n\n![Chat prompt to refactor source code and generate code\ncomments](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image15_aHR0cHM6_1750097645364.png)\n\n\n### 10. Get creative with slash commands\n\n\nWhen the Chat prompt does not know an answer to a question about the source\ncode or programming language, look into the slash commands `/explain`,\n`/refactor`, and `/tests` and how much they can help in the context.\n\n\nIn the following example, an SQL query string in C++ is created in a single\nline. To increase readability, and also add more database columns in the\nfuture, it can be helpful to change the formatting into a multi-line string.\n\n\n> std::string sql = \"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY\nKEY AUTOINCREMENT, name TEXT NOT NULL, email TEXT NOT NULL)\";\n\n\nYou can ask GitLab Duo Chat about it, for example, with the following\nquestion:\n\n\n> How to create a string in C++ using multiple lines?\n\n\nChat may answer with an explanation and optional, a source code example. In\nthis context, it can interpret the question to create a C++ string value\nwith multiple lines, for example, using the `\\n` character, assigned to a\nvariable. \n\n\nThe requirement instead is to only format the written code, and variable\nvalue assignment in multiple lines. The string value itself does not need to\ncontain a multi-line string representation. \n\n\nThere is an alternative for additional context in VS Code and the Web IDE:\nSelect the source code in question, right-click, and navigate into `GitLab\nDuo Chat > Refactor`. This opens the Chat prompt and fires the `/refactor`\ncode task immediately.\n\n\nAlthough, the code task might not bring the expected results. Refactoring a\nsingle-line SQL string can mean a lot of things: Use multiple lines for\nreadability, create constants, etc.\n\n\nCode tasks provide an option to refine the prompt. You can add more text\nafter the `/refactor` command, and instruct GitLab Duo Chat to use a\nspecific code type, algorithm, or design pattern. \n\n\nLet's try it again: Select the source code, change focus into Chat, and type\nthe following prompt, followed by `Enter`. \n\n\n> /refactor into a multi-line written string. Show different approaches for\nall C++ standards.\n\n\n![Chat prompt to refactor into a multi-line written string and\nresponse](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image17_aHR0cHM6_1750097645364.png)\n\n\n**Tip:** You can use GitLab Duo Code Suggestions to refine the source code\neven more after refactoring, or use alternative `/refactor` prompt\nrefinements.\n\n\n>/refactor into a multi-line written string, show different approaches\n\n>\n\n> /refactor into multi-line string, not using raw string literals\n\n>\n\n> /refactor into a multi-line written string. Make the table name\nparametrizable\n\n\nAn alternative approach with the `stringstream` type is shown in the [GitLab\nDuo Coffee Chat: Refactor C++ functions into OOP classes for abstract\ndatabase handling](https://www.youtube.com/watch?v=Z9EJh0J9358), [MR\ndiff](https://gitlab.com/gitlab-da/use-cases/ai/gitlab-duo-coffee-chat/gitlab-duo-coffee-chat-2024-01-23/-/commit/7ea233138aed46d77e6ce0d930dd8e10560134eb#4ce01e4c84d4b62df8eed159c2db3768ad4ef8bf_33_35). \n\n\n#### Explain vulnerabilities\n\n\nIt might not always work, but the `/explain` slash command can be asked\nabout security vulnerability explanations, too. In this example, the [C\ncode](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts/-/blob/5a5f293dfbfac7222ca4013d8f9ce9b462e4cd3a/chat/slash-commands/c/vuln.c)\ncontains multiple vulnerabilities for strcpy() buffer overflows, world\nwritable file permissions, race condition attacks, and more.\n\n\n> /explain why this code has multiple vulnerabilities\n\n\n![Chat prompt about the code's multiple\nvulnerabilities](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image20_aHR0cHM6_1750097645365.png)\n\n\n#### Refactor C code into Rust\n\n\nRust provides memory safety. You can ask Duo Chat to refactor the vulnerable\n[C\ncode](https://gitlab.com/gitlab-da/use-cases/ai/ai-workflows/gitlab-duo-prompts/-/blob/5a5f293dfbfac7222ca4013d8f9ce9b462e4cd3a/chat/slash-commands/c/vuln.c)\ninto Rust, using `/refactor into Rust`. Practice with more refined prompts\nto get better results.\n\n\n> /refactor into Rust and use high level libraries\n\n\n![Chat\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image8_aHR0cHM6_1750097645366.png)\n\n\n### Shortcuts \n\n\nGive these shortcuts a try in your environment, and practice async using\nGitLab Duo Chat.\n\n\n1. Inspect vulnerable code from CVEs, and ask what it does, and how to fix\nit, using `/explain why is this code vulnerable`. \n\n**Tip:** Import open-source projects in GitLab to take advantage of GitLab\nDuo Chat code explanations.\n\n1. Try to refactor code into new programming languages to help legacy code\nmigration plans.\n\n1. You can also try to refactor Jenkins configuration into GitLab CI/CD,\nusing `/refactor into GitLab CI/CD configuration`. \n\n\n### Fun exercises \n\n\nTry to convince Chat to behave like Clippy.\n\n![Chat\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image22_aHR0cHM6_1750097645366.png)\n\n\nAsk about GitLab's mission: \"Everyone can contribute.\"\n\n\n![Chat\nprompt](https://res.cloudinary.com/about-gitlab-com/image/upload/v1750097645/Blog/Content%20Images/Blog/Content%20Images/image33_aHR0cHM6_1750097645367.png)\n\n\n### Learn more\n\n\nThere are many different environments and challenges out there. We have\nupdated the [GitLab Duo Chat\ndocumentation](https://docs.gitlab.com/ee/user/gitlab_duo_chat.html) with\nmore practical examples, and added a new [GitLab Duo\nexamples](https://docs.gitlab.com/ee/user/gitlab_duo_examples.html) section\nwith deep dives into AI-powered DevSecOps workflows, including Chat.\n\n\n> Want to get going with GitLab Duo Chat? [Start your free trial\ntoday](https://about.gitlab.com/solutions/gitlab-duo-pro/self-managed-and-gitlab-dedicated-trial/).\n",[720,9,495,696],{"slug":4795,"featured":90,"template":700},"10-best-practices-for-using-ai-powered-gitlab-duo-chat","content:en-us:blog:10-best-practices-for-using-ai-powered-gitlab-duo-chat.yml","10 Best Practices For Using Ai Powered Gitlab Duo Chat","en-us/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat.yml","en-us/blog/10-best-practices-for-using-ai-powered-gitlab-duo-chat",24,[678,705,727,751,772,796,817,836,858],1758747471587]