"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[3085],{51512:function(e,t,o){o.d(t,{w:function(){return r}});var i="m_bb",s=o(85893),r=function(e){var t=e.offset,o=void 0===t?40:t;return(0,s.jsx)("div",{className:i,style:{height:o}})}},28962:function(e,t,o){o.d(t,{F:function(){return l}});var i=o(56338),s=o(4511),r=o(46237),n={item:"bS_3",card:"bS_dh",infoContainer:"bS_d0",text:"bS_h",footer:"bS_4",fakeLink:"bS_ck",footerText:"bS_d1",list:"bS_cl"},a=o(85893),l=function(e){var t=e.list;return(0,a.jsx)("ul",{className:n.list,children:t.map((function(e,t){return(0,a.jsx)("li",{className:n.item,children:(0,a.jsxs)(i.e,{className:n.card,href:e.link,children:[(0,a.jsx)(r.D,{level:5,className:n.title,children:e.title}),(0,a.jsx)("div",{className:n.infoContainer,children:(0,a.jsx)("p",{className:n.text,children:e.info})}),(0,a.jsxs)("div",{className:n.footer,children:[(0,a.jsx)(s.r,{className:n.fakeLink,iconPosition:"right",children:e.fakeLink}),e.footerText?(0,a.jsx)("span",{className:n.footerText,children:e.footerText}):null]})]})},t)}))})}},882:function(e,t,o){o.r(t),o.d(t,{default:function(){return T}});var i=o(74533),s=o(88586),r=o(37990),n=o(52893),a=o(93252),l=o(41752),c=o(67911),d=o(41828),h=o(51512),u=o(28962),f=o(92401),m=o(17623),g=o(25109),x=o(3468),p=o(42838),k=o(46237),w=o(76548),b=o(28447),j=o(98960),y=o(1605),L=o(42544),v=o(15218),C=o(4511),P=o(85893);function T(){return(0,P.jsxs)(P.Fragment,{children:[(0,P.jsx)(n.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/knowledgebase/crowdsourcing-methodology/share-img.jpg"}}),(0,P.jsx)(i.n,{}),(0,P.jsxs)(s.i,{children:[(0,P.jsx)(r.h,{}),(0,P.jsx)(l.p,{title:"Crowdsourcing concepts",text:"Tap into the wisdom of the crowd on a large scale"}),(0,P.jsx)(h.w,{offset:40}),(0,P.jsxs)(g.L,{children:[(0,P.jsx)(k.D,{level:3,children:"Keys to clean and accurate training data"}),(0,P.jsxs)("p",{children:["Our methodology based on years of research and unique industry expertise can help you successfully tap into the wisdom of the crowd on a large scale. If you want to efficiently use the knowledge of thousands of people to get clean and accurate data for your ML needs, follow our tips for each of these essential steps."," "]})]}),(0,P.jsx)(g.L,{children:(0,P.jsx)(c.A,{list:[{icon:(0,P.jsx)(j.z,{}),title:"1. Decomposition",text:"Break your task down into the smallest possible steps and make each one a separate task.",link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/decomposition/"}},{icon:(0,P.jsx)(v.D,{}),title:"2. Instructions",text:"The more comprehensive the instructions, the more accurate the results.",link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/instruction/"}},{icon:(0,P.jsx)(w.m,{}),title:"3. Interfaces",text:"A good interface makes it easy for users to perform the same repeated actions quickly and correctly.",link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/interface/"}},{icon:(0,P.jsx)(L.n,{}),title:"4. Quality control",text:"Carefully plan and configure a quality control system to ensure high-quality results.",link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/quality-control/"}},{icon:(0,P.jsx)(b.x,{}),title:"5. Pricing",text:(0,P.jsxs)(P.Fragment,{children:["Find the optimal price based on speed"," ",(0,P.jsx)(x.p.Desktop,{children:(0,P.jsx)("br",{})}),"and quality."]}),link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/pricing/"}},{icon:(0,P.jsx)(y.T,{}),title:"6. Results",text:"After the pool is finished, aggregate the results and check statistics.",link:{text:"Learn more",iconPosition:"right",href:"/knowledgebase/results/"}}],maxItemsInRow:3})}),(0,P.jsx)(h.w,{offset:40}),(0,P.jsx)(f.F,{leftColumn:{title:{text:"Research Benchmarks"},text:(0,P.jsxs)(P.Fragment,{children:["Quality control lies at the heart of crowdsourcing."," ",(0,P.jsx)(x.p.Desktop,{children:(0,P.jsx)("br",{})}),"Use our examples as benchmarks to achieve the described levels of quality on popular research datasets."]}),links:[{view:"action",text:"Learn more",size:"l",type:"link",url:"/knowledgebase/benchmarks/"}]}}),(0,P.jsx)(h.w,{offset:40}),(0,P.jsx)(g.L,{header:{title:"Research papers",info:(0,P.jsxs)("p",{children:["Browse through some of our"," ",(0,P.jsx)(C.r,{href:"/research/",target:"_blank",children:"Research team's"})," ","latest work."]})},children:(0,P.jsx)(u.F,{list:[{link:"https://arxiv.org/abs/2110.14990",title:"IMDB-WIKI-SbS: An Evaluation Dataset for Crowdsourced Pairwise Comparisons",info:"Our dataset has balanced distributions of age and gender using the well-known IMDB-WIKI dataset as ground truth. We describe how our dataset is built and then compare several baseline methods, indicating its suitability for model evaluation.",fakeLink:"Learn more",footerText:"NeurIPS 2021"},{link:"https://openreview.net/forum?id=3_hgF1NAXU7",title:"CrowdSpeech & Vox DIY: Benchmark Dataset for Crowdsourced Audio Transcription",info:"Domain-specific data is the crux of the successful transfer of machine learning systems from benchmarks to real life. Crowdsourcing has become one of the standard tools for cheap and time-efficient data collection for simple problems such as image classification: thanks in large part to advances in research on aggregation methods.",fakeLink:"Learn more",footerText:"NeurIPS 2021"},{link:"https://arxiv.org/abs/2109.08584",title:"A General-Purpose Crowdsourcing Computational Quality Control Toolkit for Python",info:"In this paper, we demonstrate Crowd-Kit, a general-purpose crowdsourcing computational quality control toolkit. It provides efficient implementations in Python of computational quality control algorithms for crowdsourcing, including uncertainty measures and crowd consensus methods.",fakeLink:"Learn more",footerText:"HCOMP 2021"},{link:"http://ceur-ws.org/Vol-2932/invited1.pdf",title:"VLDB 2021 Crowd Science Challenge on Aggregating Crowdsourced Audio Transcriptions",info:"This paper reviews the crowdsourced audio transcription shared task devoted to this problem and co-organized with the Crowd Science Workshop at VLDB 2021.",fakeLink:"Learn more",footerText:"VLDB 2021"},{link:"https://dl.acm.org/doi/10.1145/3394486.3403369",title:"Prediction of Hourly Earnings and Completion Time on a Crowdsourcing Platform",info:"We study the problem of predicting future hourly earnings and task completion time for a crowdsourcing platform user who sees the list of available tasks and wants to select one of them to execute.",fakeLink:"Learn more",footerText:"KDD 2020"},{link:"https://dl.acm.org/doi/10.1145/3336191.3371795",title:"Text Recognition Using Anonymous CAPTCHA Answers",info:"In this paper, we address the problem of labeling text images via CAPTCHA, where user identification is generally impossible. We propose a new algorithm to aggregate multiple guesses collected through CAPTCHA.",fakeLink:"Learn more",footerText:"WSDM 2020"}]})}),(0,P.jsx)(h.w,{offset:40}),(0,P.jsx)(g.L,{children:(0,P.jsx)(k.D,{level:3,children:"Useful resources"})}),(0,P.jsx)(g.L,{children:(0,P.jsx)(c.A,{list:[{icon:(0,P.jsx)(d.F,{}),title:"API",text:"Integrate on-demand global crowdforce & build fully automated ML pipelines.",link:{text:"Learn more",iconPosition:"right",href:"/docs/api/concepts/about.html?lang=en"}},{icon:(0,P.jsx)(m.g,{}),title:"Python library",text:"We have an open-sourced library with a client that covers all API functionalities.",link:{text:"Learn more",iconPosition:"right",href:"https://github.com/Toloka/toloka-kit#toloka-kit",target:"_blank"}},{icon:(0,P.jsx)(p.F,{}),title:"Public datasets",text:"Use our datasets for your projects or collect your own data that meets your needs.",link:{text:"Learn more",iconPosition:"right",href:"/datasets"}}],maxItemsInRow:3})}),(0,P.jsx)(f.F,{leftColumn:{title:{text:"Get started now"},text:"Let's talk about the ideal solution for your data needs.",links:[{view:"action",text:"Talk to us",size:"l",type:"link",url:"/talk-to-us"}]}}),(0,P.jsx)(h.w,{offset:40}),(0,P.jsx)(a.$,{})]})]})}}}]);