"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[218],{31451:function(e,t,n){n.d(t,{YD:function(){return u}});var i=n(67294);function s(){return s=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var i in n)Object.prototype.hasOwnProperty.call(n,i)&&(e[i]=n[i])}return e},s.apply(this,arguments)}var o=new Map,r=new WeakMap,a=0;function l(e){return Object.keys(e).sort().filter((function(t){return void 0!==e[t]})).map((function(t){return t+"_"+("root"===t?(n=e.root)?(r.has(n)||(a+=1,r.set(n,a.toString())),r.get(n)):"0":e[t]);var n})).toString()}function c(e,t,n){if(void 0===n&&(n={}),!e)return function(){};var i=function(e){var t=l(e),n=o.get(t);if(!n){var i,s=new Map,r=new IntersectionObserver((function(t){t.forEach((function(t){var n,o=t.isIntersecting&&i.some((function(e){return t.intersectionRatio>=e}));e.trackVisibility&&void 0===t.isVisible&&(t.isVisible=o),null==(n=s.get(t.target))||n.forEach((function(e){e(o,t)}))}))}),e);i=r.thresholds||(Array.isArray(e.threshold)?e.threshold:[e.threshold||0]),n={id:t,observer:r,elements:s},o.set(t,n)}return n}(n),s=i.id,r=i.observer,a=i.elements,c=a.get(e)||[];return a.has(e)||a.set(e,c),c.push(t),r.observe(e),function(){c.splice(c.indexOf(t),1),0===c.length&&(a.delete(e),r.unobserve(e)),0===a.size&&(r.disconnect(),o.delete(s))}}function d(e){return"function"!=typeof e.children}var h=function(e){var t,n;function o(t){var n;return(n=e.call(this,t)||this).node=null,n._unobserveCb=null,n.handleNode=function(e){n.node&&(n.unobserve(),e||n.props.triggerOnce||n.props.skip||n.setState({inView:!!n.props.initialInView,entry:void 0})),n.node=e||null,n.observeNode()},n.handleChange=function(e,t){e&&n.props.triggerOnce&&n.unobserve(),d(n.props)||n.setState({inView:e,entry:t}),n.props.onChange&&n.props.onChange(e,t)},n.state={inView:!!t.initialInView,entry:void 0},n}n=e,(t=o).prototype=Object.create(n.prototype),t.prototype.constructor=t,t.__proto__=n;var r=o.prototype;return r.componentDidUpdate=function(e){e.rootMargin===this.props.rootMargin&&e.root===this.props.root&&e.threshold===this.props.threshold&&e.skip===this.props.skip&&e.trackVisibility===this.props.trackVisibility&&e.delay===this.props.delay||(this.unobserve(),this.observeNode())},r.componentWillUnmount=function(){this.unobserve(),this.node=null},r.observeNode=function(){if(this.node&&!this.props.skip){var e=this.props,t=e.threshold,n=e.root,i=e.rootMargin,s=e.trackVisibility,o=e.delay;this._unobserveCb=c(this.node,this.handleChange,{threshold:t,root:n,rootMargin:i,trackVisibility:s,delay:o})}},r.unobserve=function(){this._unobserveCb&&(this._unobserveCb(),this._unobserveCb=null)},r.render=function(){if(!d(this.props)){var e=this.state,t=e.inView,n=e.entry;return this.props.children({inView:t,entry:n,ref:this.handleNode})}var o=this.props,r=o.children,a=o.as,l=o.tag,c=function(e,t){if(null==e)return{};var n,i,s={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(s[n]=e[n]);return s}(o,["children","as","tag","triggerOnce","threshold","root","rootMargin","onChange","skip","trackVisibility","delay","initialInView"]);return(0,i.createElement)(a||l||"div",s({ref:this.handleNode},c),r)},o}(i.Component);function u(e){var t=void 0===e?{}:e,n=t.threshold,s=t.delay,o=t.trackVisibility,r=t.rootMargin,a=t.root,l=t.triggerOnce,d=t.skip,h=t.initialInView,u=(0,i.useRef)(),f=(0,i.useState)({inView:!!h}),p=f[0],m=f[1],g=(0,i.useCallback)((function(e){void 0!==u.current&&(u.current(),u.current=void 0),d||e&&(u.current=c(e,(function(e,t){m({inView:e,entry:t}),t.isIntersecting&&l&&u.current&&(u.current(),u.current=void 0)}),{root:a,rootMargin:r,threshold:n,trackVisibility:o,delay:s}))}),[Array.isArray(n)?n.toString():n,a,r,l,d,o,s]);(0,i.useEffect)((function(){u.current||!p.entry||l||d||m({inView:!!h})}));var b=[g,p.inView,p.entry];return b.ref=b[0],b.inView=b[1],b.entry=b[2],b}h.displayName="InView",h.defaultProps={threshold:0,triggerOnce:!1,initialInView:!1}},92033:function(e,t,n){n.d(t,{J:function(){return I}});var i="bR_dN",s="bR_dP",o="bR_dQ",r="bR_bC",a="bR_dR",l="bR_dS",c="bR_dT",d="bR_dV",h="bR_dW",u="bR_dX",f="bR_dY",p="bR_dZ",m=n(79700),g=n.n(m),b={list:"d9_cl",item:"d9_3",button:"d9_6",activeItem:"d9_jB"},v=n(67294),w=n(55074),x=n(85893),y=function(e){var t=e.toc,n=(0,w.A)(t.map((function(e){return e.url})),t.length>0?t[0].url:void 0),i=(0,v.useCallback)((function(e){var t=document.getElementById(e.slice(1));if(t){var n=document.body.clientWidth>=1024?60:120,i=t.getBoundingClientRect();window.scrollTo({top:i.top+window.pageYOffset-i.height-n,behavior:"smooth"})}}),[]);return 0===t.length?(0,x.jsx)("div",{className:b.wrap}):(0,x.jsx)("div",{className:b.wrap,children:(0,x.jsx)("ul",{className:b.list,children:t.map((function(e){return(0,x.jsx)("li",{className:g()(b.item,e.url===n?b.activeItem:void 0),children:(0,x.jsx)("button",{className:g()("button-reset",b.button),onClick:function(){return i(e.url)},children:e.title})},e.url)}))})})},k=n(61123),j=n(4511),C=n(11467),_=n(38477),I=function(e){var t=e.children,n=e.toc,m=e.withoutShareLink,b=e.form,w=e.backLink,I=e.additionalLink,T=e.noBottomBorder,V=(0,v.useMemo)((function(){return b?(0,x.jsx)("div",{className:l,children:b}):null}),[b]);return(0,x.jsxs)("div",{className:f,children:[(0,x.jsxs)("div",{className:g()(o,n?void 0:p),children:[n&&n.length>0?(0,x.jsx)("div",{className:g()(i,T?s:void 0),children:(0,x.jsx)(y,{toc:n})}):null,m?null:(0,x.jsx)("div",{className:a,children:(0,x.jsx)(k.D,{})}),V]}),(0,x.jsxs)("div",{className:r,children:[(0,x.jsxs)(C.O,{children:[V,t]}),w||I?(0,x.jsxs)("div",{className:c,children:[w?(0,x.jsx)(j.r,{className:g()(d,"back-link"),onClick:function(){return(0,_.X)(w.href)},iconPosition:w.iconPosition,children:w.text}):null,I?(0,x.jsx)("div",{className:h,children:(0,x.jsx)(j.r,{href:I.href,className:u,iconPosition:I.iconPosition,children:I.text})}):null]}):null]})]})}},7291:function(e,t,n){n.d(t,{v:function(){return v}});var i=n(46237),s="bX_Z",o="bX_dK",r="bX_q",a="bX_h",l="bX_dS",c="bX_dd",d=n(99628),h=n(84401),u=n(19224),f=n(79700),p=n.n(f),m=n(14245),g=n(27850),b=n(85893),v=function(e){var t=e.title,n=e.text,f=e.inputPlaceholder,v=e.button,w=e.successText,x=e.sendingData,y=e.fractal,k=void 0===y?(0,b.jsx)(u.J,{}):y,j=(0,g.Q)(x),C=j.onSubmit,_=j.isSubmitted,I=j.setInputText;return(0,b.jsxs)("div",{className:p()(s,"light"),children:[t?(0,b.jsx)(i.D,{level:4,className:r,children:t}):null,(0,b.jsx)("p",{className:a,children:n}),(0,b.jsx)("form",{className:l,onSubmit:C,children:_?(0,b.jsx)(m.r,{text:w}):(0,b.jsxs)(b.Fragment,{children:[(0,b.jsx)(d.z,{view:"default",size:"m",type:"email",placeholder:f,className:o,onChange:function(e){return I(e.target.value)},autoComplete:"on",required:!0}),(0,b.jsx)(h.z,{view:v.view,type:"submit",size:v.size,children:v.text})]})}),(0,b.jsx)("div",{className:c,children:k})]})}},86457:function(e,t,n){n.d(t,{C:function(){return y}});var i=n(79700),s=n.n(i),o="bg_cV",r="bg_3",a="bg_cW",l="bg_cX",c="bg_v",d="bg_q",h="bg_bn",u="bg_cY",f="bg_j",p="bg_cZ",m="bg_b3",g="bg_c0",b="bg_c1",v=n(30759),w=n(4511),x=n(85893),y=function(e){var t=e.list,n=e.className,i=e.itemClassName,y=e.maxItemsInRow,k=e.isBigIcon,j=e.inLine,C=e.twoElementsInMobile,_=e.isSmallIcon;return(0,x.jsx)("ul",{className:s()("features-list-container",n,1===t.length||1===y?o:void 0,j?p:void 0,C?g:void 0,_?l:void 0),children:t.map((function(e,t){return(0,x.jsxs)("li",{className:s()(r,k?a:void 0,i,2===y?b:void 0,j?m:void 0,e.text||e.link||e.additional?void 0:c),children:[e.icon,(0,x.jsxs)("div",{children:[e.title?(0,x.jsx)(v.D,{level:5,className:d,children:e.title}):null,e.text?(0,x.jsx)("p",{className:h,children:e.text}):null,e.link?(0,x.jsx)("div",{className:f,children:(0,x.jsx)(w.r,{href:e.link.href,iconPosition:e.link.iconPosition,target:e.link.target,children:e.link.text})}):null,e.additional?(0,x.jsx)("div",{className:u,children:e.additional}):null]})]},t)}))})}},61123:function(e,t,n){n.d(t,{D:function(){return g}});var i={list:"fb_cl",item:"fb_3",button:"fb_6"},s=n(79700),o=n.n(s),r=n(67294),a=n(85893),l=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"16",height:"13",viewBox:"0 0 16 13",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M5.96907 12.861C5.27718 12.861 4.56503 12.8068 3.8486 12.6982C2.58677 12.5067 1.30944 11.8684 0.947991 11.6773L0 11.176L1.02636 10.8496C2.1483 10.4928 2.83079 10.2714 3.67564 9.92485C2.82969 9.52859 2.17772 8.8159 1.86388 7.89458L1.62511 7.19355L1.82091 7.22248C1.63524 7.04149 1.48864 6.85779 1.37499 6.69075C0.970818 6.09703 0.75683 5.37156 0.802362 4.74987L0.847284 4.13798L1.22606 4.27989C1.06628 3.98805 0.9508 3.67638 0.882928 3.34971C0.717524 2.55306 0.85595 1.7067 1.27282 0.966604L1.60277 0.380799L2.04405 0.893404C3.43967 2.51481 5.20736 3.47662 7.30536 3.75819C7.21967 3.18691 7.28388 2.63558 7.4964 2.14139C7.74384 1.56598 8.18402 1.07804 8.76898 0.73025C9.41867 0.344084 10.2065 0.156962 10.9874 0.203359C11.8158 0.252589 12.568 0.552809 13.1654 1.07214C13.4572 0.998835 13.6722 0.920559 13.9627 0.81478C14.1374 0.751146 14.3357 0.678895 14.5837 0.596254L15.4978 0.291666L14.9018 1.9381C14.9411 1.93491 14.9816 1.93219 15.0239 1.93043L16 1.88769L15.4231 2.65022C15.39 2.6939 15.3816 2.7063 15.3699 2.72342C15.3234 2.79118 15.2655 2.87547 14.4737 3.89809C14.2754 4.15416 14.1764 4.48767 14.1948 4.83736C14.2651 6.16621 14.0964 7.36851 13.6935 8.41073C13.3124 9.39676 12.7219 10.2427 11.9384 10.9252C10.9689 11.7696 9.73271 12.3477 8.26397 12.6433C7.54352 12.7883 6.76985 12.861 5.96907 12.861Z",fill:"#1E2126",fillOpacity:"0.5"})})},c=function(){return(0,a.jsxs)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"18",height:"18",viewBox:"0 0 18 18",fill:"none",children:[(0,a.jsx)("path",{d:"m9 14.14-1.29 1.29a3.64 3.64 0 0 1-5.14 0v0a3.64 3.64 0 0 1 0-5.14L5.14 7.7a3.64 3.64 0 0 1 5.15 0v0l.28.29",stroke:"#1E2126",strokeOpacity:".5",strokeWidth:"2.4",strokeLinecap:"round",strokeLinejoin:"round"}),(0,a.jsx)("path",{d:"m9 3.86 1.29-1.29a3.64 3.64 0 0 1 5.14 0v0a3.64 3.64 0 0 1 0 5.14l-2.57 2.58a3.64 3.64 0 0 1-5.15 0v0l-.21-.22",stroke:"#1E2126",strokeOpacity:".5",strokeWidth:"2.4",strokeLinecap:"round",strokeLinejoin:"round"})]})},d=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"8",height:"16",viewBox:"0 0 8 16",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M6.54 2.65673H8.00067V0.112728C7.74867 0.078061 6.882 6.10352e-05 5.87267 6.10352e-05C3.76667 6.10352e-05 2.324 1.32473 2.324 3.75939V6.00006H0V8.84406H2.324V16.0001H5.17333V8.84473H7.40333L7.75733 6.00073H5.17267V4.04139C5.17333 3.21939 5.39467 2.65673 6.54 2.65673Z",fill:"#1E2126",fillOpacity:"0.5"})})},h=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"16",height:"16",viewBox:"0 0 16 16",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M2.15471 0.207764C0.966598 0.207764 0 1.15446 0 2.31809C0 3.48225 0.966598 4.4293 2.15471 4.4293C3.34188 4.4293 4.30769 3.4822 4.30769 2.31809C4.30775 1.15446 3.34188 0.207764 2.15471 0.207764ZM3.43672 5.03238H0.255583C0.114399 5.03238 0 5.14816 0 5.29093V15.6292C0 15.772 0.114399 15.8878 0.255583 15.8878H3.43672C3.57791 15.8878 3.69231 15.772 3.69231 15.6292V5.29093C3.69231 5.14816 3.57791 5.03238 3.43672 5.03238ZM9.11072 6.19188C9.68593 5.57515 10.6343 5.03238 11.9172 5.03238C15.5149 5.03238 16 7.59188 16 10.0956V15.6351C16 15.7747 15.8852 15.8878 15.7433 15.8878H12.555C12.4132 15.8878 12.2984 15.7747 12.2984 15.6351V10.725C12.2984 9.35023 12.1951 8.29862 10.8963 8.29862C9.70362 8.29862 9.23894 8.9546 9.23894 10.6383V15.6351C9.23894 15.7746 9.12407 15.8877 8.98231 15.8877H5.7951C5.65333 15.8877 5.53847 15.7746 5.53847 15.6351V5.53596C5.53847 5.39649 5.65333 5.28338 5.7951 5.28338H8.85409C8.99586 5.28338 9.11072 5.39649 9.11072 5.53596V6.19188Z",fill:"#1E2126",fillOpacity:"0.5"})})},u=n(9546),f=n(47250),p=n(81547);function m(e){var t=[];for(var n in e)t.push(encodeURIComponent(n)+"="+encodeURIComponent(e[n]));return t.join("&")}var g=function(){if(p.j){var e=window.location.href,t=(0,u.F)(e),n=(0,r.useMemo)((function(){var t={url:e,utm_source:"share2"},n={u:e,src:"sp",utm_source:"share2"},i={mini:"true",url:e,utm_source:"share2"};return[{icon:(0,a.jsx)(l,{}),title:"Twitter",link:"https://twitter.com/intent/tweet?"+m(t)},{icon:(0,a.jsx)(d,{}),title:"Facebook",link:"https://www.facebook.com/sharer.php?"+m(n)},{icon:(0,a.jsx)(h,{}),title:"Linkedin",link:"https://www.linkedin.com/shareArticle?"+m(i)}]}),[e]);return(0,a.jsx)("div",{className:i.wrap,children:(0,a.jsxs)("ul",{className:i.list,children:[n.map((function(e){return(0,a.jsx)("li",{className:i.item,children:(0,a.jsxs)("a",{className:o()("button-reset",i.button),href:e.link,target:"_blank",rel:"nofollow noopener noreferrer",title:e.title,children:[e.icon,(0,a.jsx)(f.M,{children:e.title})]})},e.title)})),(0,a.jsx)("li",{className:i.item,children:(0,a.jsxs)("a",{className:o()("button-reset",i.button),href:"#",target:"_blank",rel:"nofollow noopener noreferrer",title:"Link to current page",onClick:function(e){return e.preventDefault()},ref:t,children:[(0,a.jsx)(c,{}),(0,a.jsx)(f.M,{children:"Link to current page"})]})})]})})}return null}},33543:function(e,t,n){n.d(t,{M:function(){return c}});var i={tab:"bW_cd",sliderWrap:"bW_cc",noSidePaddings:"bW_cn",defaultView:"bW_d5",dark:"bW_c3",modifiedView:"bW_d6"},s=n(79700),o=n.n(s),r=n(67294),a=n(21648),l=n(85893),c=function(e){var t=e.items,n=e.render,s=e.shouldTabsChangeView,c=void 0===s||s,d=e.theme,h=e.noSidePaddings,u=(0,a.u)(),f=u.ref,p=u.setSwiperRef,m=(0,r.useState)(0),g=m[0],b=m[1];return(0,l.jsxs)(l.Fragment,{children:[t.length>1?(0,l.jsx)("div",{className:o()(i.tab,d?i[d]:void 0,"tab-slider",c?i.modifiedView:i.defaultView,h?i.noSidePaddings:void 0),ref:f,children:t.map((function(e,t){return(0,l.jsxs)("button",{className:t===g?"active":void 0,onClick:function(){return b(t)},children:[e.icon,(0,l.jsx)("span",{children:e.title})]},t)}))}):null,(0,l.jsx)("div",{className:o()(i.sliderWrap,t.length>1?"withTabs":void 0),children:n({activeTab:g,setSwiperRef:p})})]})}},55074:function(e,t,n){n.d(t,{A:function(){return s}});var i=n(67294),s=function(e,t){var n=(0,i.useState)(t),s=n[0],o=n[1];return(0,i.useEffect)((function(){var t=document.body.clientWidth>=1024?100:150,n=new IntersectionObserver((function(e){e.forEach((function(e){!1!==e.isIntersecting&&e.boundingClientRect.y<200&&o("#"+e.target.id)}))}),{root:null,rootMargin:"-"+t+"px 0px 0px 0px",threshold:[0,.1,.2,.3,.4,.5,.6,.7,.8,.9,1]});return e.map((function(e){return document.getElementById(e.slice(1))})).filter(Boolean).forEach((function(e){n.observe(e)})),function(){n.disconnect()}}),[e,t]),s}},21648:function(e,t,n){n.d(t,{u:function(){return o}});var i=n(67294),s=n(31451),o=function(){var e=(0,i.useState)(null),t=e[0],n=e[1],o=(0,s.YD)({threshold:0,triggerOnce:!0}),r=o.ref,a=o.inView;return(0,i.useEffect)((function(){t&&(t.autoplay.stop(),a&&t.autoplay.start())}),[a,t]),{ref:r,swiperRef:t,setSwiperRef:n}}},38477:function(e,t,n){n.d(t,{X:function(){return i}});var i=function(e){document.referrer.includes(e)?window.history.back():window.open(e,"_self")}},67911:function(e,t,n){n.d(t,{A:function(){return w}});var i="F_Z",s="F_b2",o="F_b3",r="F_b4",a="F_b5",l="F_b6",c="F_b7",d="F_b8",h="F_b9",u="F_cb",f=n(79700),p=n.n(f),m=n(86457),g=n(67294),b=n(85893),v={1:u,2:d,3:h,4:void 0},w=function(e){var t=e.list,n=e.maxItemsInRow,d=void 0===n?4:n,h=e.isBigIcon,u=e.withImage,f=e.inLine,w=e.twoElementsInMobile,x=e.columnFlow,y=e.paddingsBeforeDesktop,k=e.isSmallIcon,j=(0,g.useRef)(null);return(0,g.useEffect)((function(){j.current&&x&&j.current.style.setProperty("--count",""+Math.ceil(t.length/d))}),[j]),(0,b.jsx)("div",{ref:j,className:p()(i,f?o:void 0,y?s:""),children:(0,b.jsx)(m.C,{list:t,className:p()(r,v[d],u?a:void 0,x&&4!==d?c:void 0),itemClassName:u?l:void 0,maxItemsInRow:d,isBigIcon:h,inLine:f,twoElementsInMobile:w,isSmallIcon:k})})}},66924:function(e,t,n){n.d(t,{n:function(){return s}});var i=n(85893),s=function(){return(0,i.jsxs)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"24",height:"25",viewBox:"0 0 24 25",fill:"none",children:[(0,i.jsx)("g",{clipPath:"url(#clip0_9129_114992)",children:(0,i.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M4.92861 19.5714C1.02365 15.6664 1.02373 9.33478 4.92926 5.42926C8.83478 1.52373 15.1664 1.52365 19.0714 5.42861C22.9764 9.33357 22.9763 15.6652 19.0707 19.5707C15.1652 23.4763 8.83357 23.4764 4.92861 19.5714ZM3.51439 20.9856C-1.17172 16.2995 -1.17143 8.70152 3.51504 4.01504C8.20152 -0.671427 15.7995 -0.671719 20.4856 4.01439C25.1717 8.7005 25.1714 16.2985 20.485 20.985C15.7985 25.6714 8.2005 25.6717 3.51439 20.9856ZM11.6703 16.0403C11.4839 16.2396 11.2243 16.354 10.9515 16.3571C10.6787 16.3602 10.4165 16.2518 10.2256 16.0569L6.99992 12.7633C6.61349 12.3687 6.62008 11.7356 7.01465 11.3492C7.40922 10.9627 8.04235 10.9693 8.42879 11.3639L10.9235 13.9111L15.5555 8.95973C15.9328 8.55641 16.5656 8.53533 16.9689 8.91263C17.3723 9.28993 17.3933 9.92274 17.016 10.3261L11.6703 16.0403Z",fill:"#0055D9"})}),(0,i.jsx)("defs",{children:(0,i.jsx)("clipPath",{id:"clip0_9129_114992",children:(0,i.jsx)("rect",{width:"24",height:"24",fill:"white",transform:"translate(0 0.5)"})})})]})}},75777:function(e,t,n){n.r(t),n.d(t,{default:function(){return y}});var i=n(74533),s=n(88586),o=n(37990),r=n(52893),a=n(93252),l=n(51512),c=n(3468),d=n(25109),h=n(4511),u=n(11467),f=n(7291),p=n(92033),m=n(33543),g=n(46237),b=n(88709),v=n(67911),w=n(66924),x=n(85893);function y(){return(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)(r.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/knowledgebase/quality-control/og/share-img.jpg",title:"Quality Control",description:"Carefully plan and configure a quality control system to ensure high-quality results."}}),(0,x.jsx)(i.n,{}),(0,x.jsxs)(s.i,{headerPadding:!0,children:[(0,x.jsx)(o.h,{defaultBackground:"dark"}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsxs)(d.L,{overflow:"inherit",children:[(0,x.jsx)(g.D,{level:1,children:"Quality Control"}),(0,x.jsx)("p",{children:"Carefully plan and configure a quality control system to ensure high-quality results."}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsxs)(p.J,{withoutShareLink:!0,noBottomBorder:!0,toc:[{url:"#overview",title:"Overview"},{url:"#selecting-performers",title:"Selecting Tolokers"},{url:"#synchronous-quality-control",title:"Synchronous quality control"},{url:"#asynchronous-quality-control",title:"Asynchronous quality control"}],form:(0,x.jsx)(f.v,{text:"Subscribe to Toloka News",inputPlaceholder:"Enter your email",button:{view:"action",text:"Subscribe",size:"l"},sendingData:{address:"/api/sender/subscribe",listId:"0ASPRWP3-H6V"},successText:"Thank you for subscribing"}),children:[(0,x.jsx)(u.V.Video,{isFrame:!0,src:"https://www.youtube.com/embed/kJU3ZNZ7VdE",hasBorder:!0}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsxs)("p",{id:"overview",children:["Quality control is essential. You have a large crowd of people completing your tasks, and you want to make sure that they understand the instructions, they're paying attention, and they aren't trying to use bots. Use a combination of strategies to manage the crowd and achieve the best quality.",(0,x.jsx)("br",{}),(0,x.jsx)(c.p.Desktop,{children:(0,x.jsx)("br",{})}),"Designing quality control is a multi-stage process. It starts with decomposing a task, writing instructions and designing a clear interface. All these things help to eliminate misunderstandings and guide the Toloker through the task. The next steps are:"]}),(0,x.jsx)(v.A,{maxItemsInRow:2,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"Pre-filtering Tolokers to match project goals"},{icon:(0,x.jsx)(w.n,{}),additional:"Using an entrance test to check if the guidelines were understood"},{icon:(0,x.jsx)(w.n,{}),additional:"Creating a training project that explains all necessary rules"},{icon:(0,x.jsx)(w.n,{}),additional:"Checking the quality of responses"},{icon:(0,x.jsx)(w.n,{}),additional:"Tracking behavior in the project"},{icon:(0,x.jsx)(w.n,{}),additional:"Using smart response aggregation"}]}),(0,x.jsx)("p",{children:"The results of all these checks can be converted into specific Toloker attributes that reflect their quality and behavior."}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(u.V.Image,{imageId:"yQd3C_FWN6ePlryhERiM"}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(u.V.Line,{}),(0,x.jsx)(l.w,{offset:60}),(0,x.jsx)(g.D,{level:2,styleAs:4,id:"selecting-performers",children:"Selecting Tolokers"}),(0,x.jsx)("p",{children:"Though it is not a quality control mechanism in a literal sense, this is a very important step and the best investment in terms of both data quality and requester's time spent. You can create a foundation for stable data quality by explaining the guidelines and testing Tolokers before they get access to the project. Create a comprehensive and clear training set and a corresponding test so that you can recruit new Tolokers whenever necessary, without wasting your time on manual selection."}),(0,x.jsx)(m.M,{noSidePaddings:!0,shouldTabsChangeView:!1,items:[{title:"Pre-filtering"},{title:"Training"},{title:"Entrance testing"}],render:function(e){var t=e.activeTab;return[(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"Pre-filtering Tolokers means offering the task only to people who have certain attributes necessary for the task. Different crowdsourcing platforms may offer different filters, but most popular are location, age, gender, known languages and devices. If you have good reason to believe that some of these properties will affect task performance, try using these filters. Here are some examples when filters might be useful:"}),(0,x.jsx)(v.A,{maxItemsInRow:2,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"If your project involves working with content in a certain language, give access only to users with corresponding language skills."},{icon:(0,x.jsx)(w.n,{}),additional:"Using an entrance test to check if the guidelines were understood"},{icon:(0,x.jsx)(w.n,{}),additional:"If your project involves UX testing for a product that has an age-based target audience, send tasks to Tolokers in the same age group."}]}),(0,x.jsxs)("p",{children:["For more information on filters available on Toloka, see the"," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/filters.html",target:"_blank",children:"docs."},1)]})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"After pre-filtering, you need to teach Tolokers to apply the rules and guidelines to real-life tasks. Create a training set of tasks. Training sets consist of tasks with hints that explain how to do the task and why. Here are some suggestions for creating effective training projects:"}),(0,x.jsx)(v.A,{maxItemsInRow:2,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"Include examples for all the rules and guidelines in your instructions, even the simplest ones."},{icon:(0,x.jsx)(w.n,{}),additional:"In the comments, explain why a certain response is appropriate with a reference to the instructions. Don't just state the correct answer."},{icon:(0,x.jsx)(w.n,{}),additional:"Don't make it too long - just cover all necessary cases."}]}),(0,x.jsxs)("p",{children:["There's a special type of pool in Toloka aimed solely for entrance training."," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/train.html",target:"_blank",children:"Learn more"},2)," ","about the settings available."]})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"After a Toloker finishes the training, redirect them to a testing set, which is a set of tasks with known answers. After the Toloker completes the test, you can calculate the percentage of correct answers and decide if it's high enough to continue working on the project. Here are some suggestions for setting up a test:"}),(0,x.jsx)(v.A,{maxItemsInRow:2,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"Testing tasks should match the topics and complexity of the training."},{icon:(0,x.jsx)(w.n,{}),additional:"It's better to create several versions of the test and alternate between them, so that it will be more difficult for Tolokers to cheat and share correct answers with each other."},{icon:(0,x.jsx)(w.n,{}),additional:"Testing tasks should be of the highest quality because you are using them to make important decisions about Tolokers. Pay attention to examples where Tolokers make a lot of mistakes: it's a sign that either something's wrong with the task or the concept wasn't explained well enough in the instructions."}]}),(0,x.jsxs)("p",{children:["A testing set is technically a pool that only consists of control tasks. See the step-by-step description of adding control tasks and quality calculation rules in the"," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/goldenset.html",target:"_blank",children:"docs."},3)]})]})][t]}}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(u.V.Line,{}),(0,x.jsx)(l.w,{offset:60}),(0,x.jsx)(g.D,{level:2,styleAs:4,id:"synchronous-quality-control",children:"Synchronous quality control"}),(0,x.jsx)("p",{children:"Most quality control methods are designed to control crowd behavior and assessments with the smallest delay possible. It is important to detect low-quality annotators quickly enough before they produce too much useless data and too much money is spent. The following sections describe some of the popular control mechanisms available on crowdsourcing platforms."}),(0,x.jsx)(m.M,{noSidePaddings:!0,shouldTabsChangeView:!1,items:[{title:"Behavior checks"},{title:"Assignment quality checks"},{title:"A few more notes on quality checks"}],render:function(e){var t=e.activeTab;return[(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"Checks like these are designed to reveal if a person regularly demonstrates suspicious behavior, like clicking through tasks too quickly and inattentively. Here are some popular approaches to control bot-like behavior:"}),(0,x.jsx)(v.A,{maxItemsInRow:1,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:" CAPTCHA."})," This is a well-known tool that helps filter out bots by displaying an image with a text that a user needs to decipher and enter. It's a great basic mechanism, but it's better not to rely solely on it because it only helps to filter out bots or users who don't pay any attention at all. Besides, regular users also tend to make CAPTCHA mistakes because CAPTCHAs may be complicated or just irritating. Best practice here is to ban users only after a certain number of missed CAPTCHAs in a row (more than one)."]})},{icon:(0,x.jsx)(w.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Speed monitoring."})," For every crowd-based task there's a reasonable amount of time it takes to understand the task and make an assessment. If a certain user regularly closes tasks much faster than the average time, they're probably just clicking through the tasks without looking at them carefully. The suspicious speed is different for every project, but it can be defined as 10-20% of the normal speed in the project. If a Toloker submits a string of tasks under this limit, it's better to single them out and ban them from the project."]})},{icon:(0,x.jsx)(w.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Checking for certain actions."})," A crowd task is basically an interface that allows you to implement any controls or checks based on Javascript. If a task involves interaction with content like playing a media file, visiting a link, or typing in text, these interactions can be checked and analyzed. As discussed in the Interfaces section, some of these checks can be transformed into warnings. Another option is to keep a record of Toloker actions and calculate attributes based on how diligently a user handles task content."]})}]})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"For most tasks it's not enough to just identify users who mindlessly blast through tasks — it's also important to check if those who are performing diligently enough are also following all the guidelines and producing quality data. Checking assignment quality is dependent on a benchmark to compare the answers to. There are two major approaches to establishing a benchmark: majority vote and expert opinion."}),(0,x.jsx)(v.A,{maxItemsInRow:1,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Majority vote"})," is based on task overlap. Overlap is when several users complete the same assignment. After all assessments are submitted, you can find which answer was selected by most users and use it as a benchmark. By comparing a particular Toloker's answers to benchmarks like these, you can calculate the percentage of cases when their opinion contradicts the opinion of the majority. This percentage can help you detect unstable quality in specific individuals. However, majority vote mechanisms can sometimes discriminate against unusually attentive people who submit correct but unobvious answers. Majority vote calculations are also prone to being skewed by spam."]})},{icon:(0,x.jsx)(w.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Control tasks"})," (a.k.a. golden sets or honeypots) solve both of these problems. A control task is a task with a known correct answer. Tolokers get these tasks mixed in with their general task flow, without knowing that it entails a special check. By adding control tasks to your project, you can determine the percentage of correct responses for the project as a whole, and the percentage of correct responses for individual users."]})}]}),(0,x.jsxs)("p",{children:["Toloka allows to set up"," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/control.html",target:"_blank",children:"these and additional quality control mechanisms"},4)," ","as independent settings or quality control presets. They are available both on pool and project levels."]})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("br",{}),(0,x.jsx)(g.D,{level:5,children:" Which overlap should I choose?"}),(0,x.jsxs)("p",{children:["Research shows that the optimal overlap for task assignment is 3 to 5. Beyond this, quality increases only slightly while price increases significantly. Be sure to check the section about"," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/dynamic-overlap.html",target:"_blank",children:"incremental relabeling,"},5)," ","which is a smart way to control costs on overlap."]}),(0,x.jsx)("br",{}),(0,x.jsxs)(g.D,{level:5,children:[" ","How many control tasks should I add?"]}),(0,x.jsx)("p",{children:"If a task set consists of several hundred tasks, then 10% of the pool should be control tasks. If the set contains thousands of tasks, just 1% is enough."}),(0,x.jsx)("br",{}),(0,x.jsx)(g.D,{level:5,children:" How do I create control tasks?"}),(0,x.jsx)("p",{children:"There are two ways to create control tasks:"}),(0,x.jsx)(v.A,{maxItemsInRow:1,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:'Send the tasks to a "trusted" crowd. Select Tolokers who are reliable in delivering high-quality results. Then create a separate project for them to generate correct responses for tasks and launch it with high overlap. Don\'t forget to use additional quality control methods, just in case.'},{icon:(0,x.jsx)(w.n,{}),additional:"Choose experts at your company who can label the data well. Have them generate control tasks to use for checking the quality of Tolokers' responses. In other words, 10% of the tasks are completed by an internal team and used for controlling 90% of the entire labeling process."}]}),(0,x.jsx)("br",{}),(0,x.jsxs)(g.D,{level:5,children:[" ","How can I maintain the quality of control tasks?"]}),(0,x.jsxs)("p",{children:[" ","First, here's a tip: if the quality or labeling speed is low, check the control tasks. Perhaps they contain incorrect, outdated or unclear examples. Control tasks need special maintenance because the project's quality metrics and individual Toloker' quality depend on them. There are two best practices:"]}),(0,x.jsx)(v.A,{maxItemsInRow:1,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"Get rid of old control tasks and replace them with new ones."},{icon:(0,x.jsx)(w.n,{}),additional:"Check for suspicious control tasks where Tolokers constantly make mistakes. The task may contain a mistake or a corresponding guideline may be unclear."}]}),(0,x.jsx)("p",{children:"There is one more important feature of a control task set: if possible, classes should be represented in the control set in proportions similar to the classes in the general pool of tasks. Let's say you need to determine the type of accommodations on a hotel aggregator website: family, business, casual or luxury. Luxury accommodations make up just 10% of the main pool, but performers see them in every second control task. As a result, you will not be able to check whether the Tolokers correctly mark the other types of accommodations, and you risk getting poor results with noisy data."})," "]})][t]}}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(u.V.Line,{}),(0,x.jsx)(l.w,{offset:60}),(0,x.jsx)(g.D,{level:2,styleAs:4,id:"asynchronous-quality-control",children:"Asynchronous quality control"}),(0,x.jsx)("p",{}),(0,x.jsx)(m.M,{noSidePaddings:!0,shouldTabsChangeView:!1,items:[{title:"Assignment review"},{title:"Smart aggregation methods"}],render:function(e){var t=e.activeTab;return[(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"Synchronous quality control methods are only applicable to tasks where there's a single correct answer. But not all crowd tasks are like that: some demand a creative approach or content processing and can have a variety of correct solutions. Tasks like these can be checked via assignment review. There are two possible ways to review a task:"}),(0,x.jsx)(v.A,{maxItemsInRow:1,inLine:!0,list:[{icon:(0,x.jsx)(w.n,{}),additional:"The review can be done by someone on the requester's side. A crowdsourcing platform may even have this option in its UI. But this is only a viable option for very small data volumes (or for requesters with unlimited resources)."},{icon:(0,x.jsx)(w.n,{}),additional:"Other Tolokers can do the review. This requires starting a new project using marked data that is transferred from the first project and asking Tolokers whether the task was completed correctly. After verification, incorrect tasks are transferred back for evaluation, payments ar made for correctly completed tasks, and individual quality can be calculated."}]})]}),(0,x.jsx)(x.Fragment,{children:(0,x.jsxs)("p",{children:["Using all data about Tolokers' individual quality and their approach to certain tasks can become a basis for smart quality enhancement via"," ",(0,x.jsx)(h.r,{href:"/knowledgebase/aggregation",target:"_blank",children:"smart aggregation."},6)," "]})})][t]}}),(0,x.jsx)(u.V.Image,{imageId:"toHWlpq0gxJ5IkCPSJOw"}),(0,x.jsxs)("p",{children:["See the"," ",(0,x.jsx)(h.r,{href:"/docs/guide/concepts/reviewing-assignments.html",target:"_blank",children:"docs"},7)," ","for a step-by-step description of assignment review settings."]})]})]}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(a.$,{}),(0,x.jsx)(b.f,{})]})]})}}}]);