-
Notifications
You must be signed in to change notification settings - Fork 8
/
Copy pathPre-test-Phase.html
629 lines (550 loc) · 35.5 KB
/
Pre-test-Phase.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head>
<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />
<title>Pre Test Phase</title>
<script src="libs/jquery-1.11.3/jquery.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="libs/bootstrap-3.3.5/css/flatly.min.css" rel="stylesheet" />
<script src="libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<script src="libs/navigation-1.1/tabsets.js"></script>
<meta name="robots" content="index, follow">
<title>IFHS: Integrated Framework for Household Survey</title>
<link rel="stylesheet" href="./include/ifhs2.css" />
<script type="text/javascript" src="./libs/zeroclipboard-2.2.0/ZeroClipboard.js"></script>
<link rel="stylesheet" href="./libs/colorbox-1.6.1/colorbox.css" />
<script type="text/javascript" src="./libs/colorbox-1.6.1/jquery.colorbox-min.js"></script>
<!--- favicon --->
<link rel="apple-touch-icon" sizes="57x57" href="./images/favicon/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="./images/favicon/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="./images/favicon/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="./images/favicon/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="./images/favicon/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="./images/favicon/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="./images/favicon/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="./images/favicon/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="./images/favicon/apple-icon-180x180.png">
<link rel="icon" type="image/png" sizes="192x192" href="./images/favicon/android-icon-192x192.png">
<link rel="icon" type="image/png" sizes="32x32" href="./images/favicon/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="96x96" href="./images/favicon/favicon-96x96.png">
<link rel="icon" type="image/png" sizes="16x16" href="./images/favicon/favicon-16x16.png">
<link rel="manifest" href="./images/favicon/manifest.json">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="./images/favicon/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
<style type="text/css">
h1 {
font-size: 34px;
}
h1.title {
font-size: 38px;
}
h2 {
font-size: 30px;
}
h3 {
font-size: 24px;
}
h4 {
font-size: 18px;
}
h5 {
font-size: 16px;
}
h6 {
font-size: 12px;
}
.table th:not([align]) {
text-align: left;
}
</style>
<style type = "text/css">
.main-container {
max-width: 940px;
margin-left: auto;
margin-right: auto;
}
code {
color: inherit;
background-color: rgba(0, 0, 0, 0.04);
}
img {
max-width:100%;
}
.tabbed-pane {
padding-top: 12px;
}
.html-widget {
margin-bottom: 20px;
}
button.code-folding-btn:focus {
outline: none;
}
summary {
display: list-item;
}
</style>
<!-- tabsets -->
<style type="text/css">
.tabset-dropdown > .nav-tabs {
display: inline-table;
max-height: 500px;
min-height: 44px;
overflow-y: auto;
background: white;
border: 1px solid #ddd;
border-radius: 4px;
}
.tabset-dropdown > .nav-tabs > li.active:before {
content: "";
font-family: 'Glyphicons Halflings';
display: inline-block;
padding: 10px;
border-right: 1px solid #ddd;
}
.tabset-dropdown > .nav-tabs.nav-tabs-open > li.active:before {
content: "";
border: none;
}
.tabset-dropdown > .nav-tabs.nav-tabs-open:before {
content: "";
font-family: 'Glyphicons Halflings';
display: inline-block;
padding: 10px;
border-right: 1px solid #ddd;
}
.tabset-dropdown > .nav-tabs > li.active {
display: block;
}
.tabset-dropdown > .nav-tabs > li > a,
.tabset-dropdown > .nav-tabs > li > a:focus,
.tabset-dropdown > .nav-tabs > li > a:hover {
border: none;
display: inline-block;
border-radius: 4px;
}
.tabset-dropdown > .nav-tabs.nav-tabs-open > li {
display: block;
float: none;
}
.tabset-dropdown > .nav-tabs > li {
display: none;
}
</style>
<!-- code folding -->
</head>
<body>
<div class="container-fluid main-container">
<nav>
<div class="navbar navbar-default navbar-fixed-top" role="navigation">
<div class="container">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="index.html">Integrated Framework for Household Survey</a>
</div>
<div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li class="dropdown">
<a href="design" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Design <span class="caret"></span></a>
<ul class="dropdown-menu multi-column columns-3" role="menu" id="menu_design">
<div class="row">
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Planning</li>
<li><a href="Assessment-Project-Document.html">Assessment Project Document</a></li>
<li><a href="Memorandum-of-Understanding.html">Memorandum of Understanding</a></li>
<li><a href="Terms-of-Reference-for-Assessment-Focal-Point.html">Terms of Reference for Assessment Focal Point</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Methodology</li>
<li><a href="Sampling.html">Sampling</a></li>
<li><a href="Interview.html">Interview approach</a></li>
<li><a href="Pre-Assessment.html">Pre-Assessment</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Form</li>
<li ><a href="protection-Topics.html">Protection Topics</a></li>
<li ><a href="Module-questions.html">Questions Modules</a></li>
<li ><a href="Guidelines-for-Customisation.html">Guidelines for Customisation</a></li>
</ul>
</div>
</div>
</ul>
</li>
<li class="dropdown">
<a href="collect" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Collect <span class="caret"></span></a>
<ul class="dropdown-menu multi-column columns-3" role="menu" id="menu_collect">
<div class="row">
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Preparing for fieldwork</li>
<li><a href="Configure-forms.html">Configure forms</a></li>
<li><a href="Pre-test-Phase.html">Pre-test Phase</a></li>
<li><a href="Fieldwork-Training-Agenda.html">Fieldwork Training and Agenda</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Using KoboToolBox</li>
<li ><a href="Data-Protection-Impact-Assessment.html">Data Protection Impact Assessment</a></li>
<li ><a href="Server-Configuration.html">Server Configuration</a></li>
<li ><a href="Data-Entry.html">Data Entry</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Fieldwork manual</li>
<li ><a href="Instructions-for-Interviewers.html">Instructions for Interviewers</a></li>
<li ><a href="Instructions-for-Supervisors-Editors.html">Instructions for Supervisors and Editors</a></li>
<li ><a href="Instructions-for-Managers.html">Instructions for Managers</a></li>
</ul>
</div>
</div>
</ul>
</li>
<li class="dropdown">
<a href="analyse" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Analyse <span class="caret"></span></a>
<ul class="dropdown-menu multi-column columns-3" role="menu" id="menu_analyse">
<div class="row">
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Analytics Steps</li>
<li ><a href="Clean-Anonymize.html">Clean & Anonymize</a></li>
<li ><a href="Describe.html">Describe</a></li>
<li ><a href="Discover.html">Discover</a></li>
<li ><a href="Predict.html">Predict</a></li>
<li ><a href="Advise.html">Advise</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Analysis Process</li>
<li ><a href="Data-Crunching.html">Data Crunching</a></li>
<li ><a href="Analysis-Workshop.html">Analysis Workshop</a></li>
<li ><a href="Model-for-Final-Report.html">Model for Final Report</a></li>
</ul>
</div>
<div class="col-sm-4">
<ul class="multi-column-dropdown">
<li class="dropdown-header">Communication</li>
<li><a href="Slides-Infographics.html">Slides & Infographics</a></li>
<li><a href="Microdata.html">Sharing microdata for social scientist</a></li>
<li><a href="Open-Data.html">Open Data</a></li>
</ul>
</div>
</div>
</ul>
</li>
<li><a href="Integrated-framework-household-survey.pdf">PDF</a></li>
</ul>
<form id="rechercher" class="navbar-form navbar-right" role="search" style="padding-top: 5px;" method="get" action="https://tontonroger.org/">
<div class="form-group">
<input name="q" type="text" class="form-control input-sm" placeholder="Search">
</div>
<button type="submit" class="btn btn-default btn-sm" name="Search">
<span class="glyphicon glyphicon-search" aria-hidden="true"></span>
</button>
</form>
</div><!--/.nav-collapse -->
</div><!--/.container -->
</div><!--/.navbar -->
</nav>
<div class="row">
<div class="col-sm-9" role="main">
<article>
<div class="fluid-row" id="header">
<h1 class="title toc-ignore">Pre Test Phase</h1>
</div>
<div id="TOC">
<ul>
<li><a href="#what-is-and-why-pre-testing">What is and why “pre-testing”?</a></li>
<li><a href="#objectives">Objectives</a></li>
<li><a href="#document-the-organisation-of-the-pre-test">Document the organisation of the Pre-test</a></li>
<li><a href="#how-many-respondents">How many respondents?</a></li>
<li><a href="#how-to-observe-pre-test">How to observe pre-test?</a></li>
<li><a href="#pre-test-results-and-recommendations">Pre-test Results and Recommendations</a></li>
<li><a href="#validity-and-reliability-guidelines">Validity and Reliability guidelines</a></li>
<li><a href="#pre-piloting">Pre-piloting</a></li>
<li><a href="#piloting">Piloting</a><ul>
<li><a href="#re-test-phase">Re-test phase</a></li>
</ul></li>
<li><a href="#post-piloting">Post-Piloting</a></li>
<li><a href="#refining-and-retuning">Refining and retuning</a></li>
</ul>
</div>
<div id="what-is-and-why-pre-testing" class="section level2">
<h2>What is and why “pre-testing”?</h2>
<p>Testing materials before they are used in live surveys allows to make certain that questionnaire is accurate, non-leading and reliable. This is often carried out at a late stage and using finished materials, i.e. the final version of a the questionnaire.</p>
<p><strong>Considerable professional and financial investment is therefore at stake</strong>, and pre-tests can constitute some of the most difficult and contentious step within a household survey project. People often think that testing a survey takes a long time. They think they don’t have the time or resources for it, and so they end up just running the survey without any testing. This is a big mistake. Even testing with one person is better than no testing at all. So if you don’t have the time or resources to do everything in this guide, just do as much as you can with what you have available.</p>
<p>As a good practice, the entire questionnaire should be submitted to a number of procedures to ensure there are no questions likely to test anything but respondent real status or opinion. There are <a href="https://academic.oup.com/poq/article/68/1/109/1855073/Methods-for-Testing-and-Evaluating-Survey">extensive guidelines</a> on this subject, belwo is a summary of the main points to look at.</p>
</div>
<div id="objectives" class="section level2">
<h2>Objectives</h2>
<p>The overall objectives and the focus of the pre-test, together with the results shoudl be properly documented before starting it.</p>
<ul>
<li><p>Questions are clear i.e. respondents do not <strong>misinterpret</strong> the questions (questions are not ambiguous or difficult to understand);</p></li>
<li><p><strong>Response categories</strong> are comprehensive and adequate for the assessed population; any answer falling into the “other (specify)” category of a multiple choice question and that constitutes about 5 percent or more of all answers to that question should be considered as a serious candidate for a separate answer category of its own. New codes for common answers that were not included in the original questionnaires will need to be created</p></li>
<li><p><strong>Flow of questions</strong> within the questionnaires is adequate;</p></li>
<li><p>Difficult or <strong>sensitive questions/modules</strong> are identified so that extra training can focus on these questions during the fieldworker training;</p></li>
<li><p>Translations are accurate, ie. <strong>translated questionnaires</strong> are working correctly. Changes in wording or improved translation will need to be incorporated when required;</p></li>
<li><p>Interviewer <strong>instructions</strong> in the Instructions for Interviewers, as well respondent <strong>informed consent</strong> in questionnaire are clear and sufficient;</p></li>
<li><p>Average <strong>duration of interviews</strong> is calculated in order to plan the fieldwork and the daily workload per interviewer/team</p></li>
</ul>
</div>
<div id="document-the-organisation-of-the-pre-test" class="section level2">
<h2>Document the organisation of the Pre-test</h2>
<ul>
<li><p><strong>Clusters/Number of interviews selected for pre-test</strong>: Describe the pre-test locations where were the households located for the pre-test, how and why were these locations selected, etc. Note that often, what is important during the pre-test is more the quality of the observation within each interview than the number of interviews itself.</p></li>
<li><p><strong>Personnel</strong>: Present the trainers and interviewers (trainees) of the pre-test. Include information on the future involvement of the participants in the rest of the assessment process.</p></li>
<li><p><strong>Training</strong>: List the dates and content of the pre-test training, as well as how it was organised. Some detail is useful on agenda and training methodology, as it can serve as lessons for the main training. Include other details as relevant: Venue, recommendations for main training, etc.</p></li>
<li><p><strong>Fieldwork</strong>: Provide the dates of actual pre-test fieldwork. Also detail on organisation (logistics, teams, areas, etc.) is very useful.</p></li>
<li><p><strong>Findings</strong>: Describe how the observations from the pre-test were collected and discussed and what process for making changes to the final questionnaires was used.</p></li>
</ul>
</div>
<div id="how-many-respondents" class="section level2">
<h2>How many respondents?</h2>
<p>As a general rule, one should aim to pretest a form with at least 5 to 10 respondents and 2 different interviewers. If the survey is more complex, <a href="https://www.researchgate.net/publication/263815867_Sample_size_for_pre-tests_of_questionnaires">as studied by reasearcher</a>, a <strong>sample of 30 respondents is recommended</strong>.</p>
<p>Even with this small number of people, a surprisingly large number of improvements can be made. Try to get within those 5 to 10 respondents a range of different people who are representative of the target group for the questionnaire. Usually, most of respondents will have the same problems with the survey, so even with such small number of people, it should be possible to identify most of the major issues. Adding more people might identify some additional smaller issues, but it also makes pretesting more time consuming and costly.</p>
</div>
<div id="how-to-observe-pre-test" class="section level2">
<h2>How to observe pre-test?</h2>
<p>Note that for the pre-test, it’s important to have 2 persons supporting the interview. One regular enumerator and an additional person to take note of each observation during the interview.</p>
<ul>
<li><p>Respondents should be asked to complete the survey while <strong>“thinking out loud”</strong>. They should tell you exactly what comes into their mind so that the observer can take notes on everything they say. This which can include paraphrasing, providing retrospective thinking or providing judgments of their confidence in what each question means.</p></li>
<li><p>The observer should look for places where the respondent <strong>hesitate or make mistakes</strong>.</p></li>
<li><p>A <strong>debriefing</strong> can be organised after each pre-test interview to ensure that potential additional observations from the enumerator are also inclued in the observation notes.</p></li>
</ul>
</div>
<div id="pre-test-results-and-recommendations" class="section level2">
<h2>Pre-test Results and Recommendations</h2>
<p>Relating to the objectives listed above, this section should include findings from the actual data gathered as well as the qualitative findings from the pre-test, including those obtained from discussions with interviewers after the pre-test fieldwork concluded.</p>
<ul>
<li><p><strong>Questionnaire:</strong><br />
This section is the main output of the Pre-test Report. The use of the table below is recommended. Please add all modules in the questionnaire. Make sure that all suggested changes are listed and that evidence is provided for final decisions. Please include observations on all country-specific modules and questions.</p></li>
<li><p><strong>Instructions:</strong><br />
Describe and list any changes or additions required in the Instructions for Interviewers as well as those introduced in the Instructions for Supervisors and Editors. Such changes typically involve translation issues, instructions for country-specific questions, but also for country-specific response categories. Appropriate corrections are incredibly helpful and will especially inform the main field work training. Note that instruction can be introduced as additionnal <em>hint</em> within each question.</p></li>
<li><p><strong>Average duration of interviews:</strong><br />
Calculate the average duration of interview for each questionnaire using the data collected in the pre-test. Typically, as interviewers become more familiar with the tools, this time will decrease and therefore a realistic duration should be proposed and included in the introductory sentences on the cover pages of the questionnaires.</p></li>
<li><p><strong>Interview process considerations:</strong><br />
Describe and address the observations from the pre-test that relate to interviewing that will be relevant for training and monitoring in the main field work (for example issues in approaching households, dealing with sensitive module and questions, flow of field work, roles and responsibilities, etc.)</p></li>
<li><p><strong>Assessment process considerations:</strong><br />
Describe here the observations, suggestions, and decisions related to the assessment planning and next steps for finalising the questionnaire (training contents/agenda, logistics, staff, support, etc.)</p></li>
</ul>
</div>
<div id="validity-and-reliability-guidelines" class="section level2">
<h2>Validity and Reliability guidelines</h2>
<p>The purpose of these guidelines is to provide an overview of the statistical processes needed to test the validity and reliability of any questionnaire. Validity is the extent to which the inferences or decisions we make by the questionnaire are meaningful, appropriate, and useful. In other words, a questionnaire is said to be valid to the extent that it measures what it is supposed to measures or can be used for the purpose for which it is intended. Validity alone is not enough to consider the questionnaire useful. A valid questionnaire should also be reliable. Statistically, a reliable questionnaire should produce consistent, replicable outcomes. There are several aspects of reliability and validity that need to be considered before arriving at a final distributable version of the questionnaire. This exercise is expected to take four weeks and is a critical step to ensure data quality.</p>
</div>
<div id="pre-piloting" class="section level2">
<h2>Pre-piloting</h2>
<p>There are specific steps to be taken and areas to be covered before piloting the questionnaire:</p>
<p>1- The dimensions that will be measured by the questionnaire should be identified: the purpose of questionnaires is to collect data and to measure aspects that cannot be measured directly. In questionnaire design, the common practice is to assume that several dimensions are measured, and groups of questions measure different dimensions. For example, socioeconomic indicators and income/expenses related questions can be used as a proxy to profile individuals or households financially. This group of questions we can call it the socioecomonic dimension. As part of the validation process, these groups (dimensions) should be pre-defined before the piloting. For example Q1 – Q5 measure dimension A, Q6 – Q10 measure dimension B, etc. If the there is an ambiguity in some indicators as where they belong e.g., we may consider them in more than one dimenison or we are not sure where to include these indicators, then a statistical tests should be applied after the piloting to check the tool dimensionality (Priciple Component Analysis, Factor Analysis, Structural Equation Modelling, or Item Reponse Theory).</p>
<p>2- The questionnaire is face and content validated: Face validation is the process of assessing the relevance of the questions subjectively by experts. An additional aspect is to evaluate covering all facets of a given dimension, content validity. At least two external ‘experts’ (should not be from the design team) have reviewed the questionnaire and approved the final piloting form. The common practice is to ask each ‘expert’ as an example, two officers from the protection team to rate the relevance of each question to the dimension being measured and the objectives of the questionnaire. The raters identify each question by a score of +1 (favourable) or 0 (unfavourable). Review the questionnaire and adjust it to the experts’ feedback, then return it for another round of rating. This process can be repeated until an agreement is reached on a final version.</p>
<p>3- The translation from English to any other language should be validated through translate-re-translate process: any version of the questionnaire that is not in English should be translated to for instance to Arabic by a second party (different than the questionnaire designers), then the translated version is translated back to English by a third party (different than the previous two). The re-translated version should be compared with the original one to detect any discrepancies. Any significant issues should be solved by working with translators to assess using the appropriate words and phrases to keep the meaning consistent.</p>
</div>
<div id="piloting" class="section level2">
<h2>Piloting</h2>
<p>The piloting phase consists of two stages, and together they are referred to as the test-retest. The test-retest consists of asking the same participants to answer the same questionnaire twice over a predefined period. This process is necessary to test the questionnaires consistency over time (temporal validity) and reliability as previously discussed. The test phase includes:</p>
<p>1- Identifying the medium of contact: phone interview or online link to the questionnaire.</p>
<p>2- You are obtaining consent from the participants. The consent should state the nature of the study, the voluntary nature of participation, what will be done with the information, if it will be shared with any other party, etc.</p>
<p>3- Clearly stating to them that they will be contacted again within maximum 3-4 weeks to do the same questionnaire again.</p>
<div id="re-test-phase" class="section level3">
<h3>Re-test phase</h3>
<p>1- The questionnaire should be sent again within 12-13 days from the first completion.</p>
<p>2- If the questionnaire in the pilot phase is filled online, a reminder through e-mail or text can be sent after 3-4 days maximum after the period of 12-13 days.</p>
<p>3- After 21 days from the initial distribution, the re-test should be finished.</p>
<p>4- The final collection should not exceed the period of 21 days.</p>
</div>
</div>
<div id="post-piloting" class="section level2">
<h2>Post-Piloting</h2>
<p>After the collection of the data from the test-re-test phases, the reliability and validity statistical tests are applied to measure:</p>
<p>1- Statistical reliability: the overall consistency of a measure. A measure is said to have a high reliability if it produces similar results under consistent conditions. Statistical reliability assumes that refugees with similar profiles should to some extent give similar answers. For example, refugees with similar characteristics (socio-economic statuses, living conditions, etc.) are expected to answer questions in the same trend, to have a common outcome. If refugees with similar characteristics are answering differently, then it indicates the characteristics used are not useful to create profiles. This can be tested through Spearman’s correlation (ρ), Cohen’s Kappa, and the overall reliability of the questionnaire by Cronbach’s Alpha.</p>
<p>2- Face and content validity are discussed before. (See Pre-piloting section)</p>
<p>3- Predictive validity: It assesses the ability of the questionnaire (instrument) to forecast future events, behaviour, attitudes or outcomes.</p>
<p>4- Construct Validity: is the degree to which an instrument measures the trait or theoretical construct that it is intended to measure. It does not have a criterion for comparison; rather it utilizes a hypothetical construct for comparison. This part of validation is the most complex and difficult to test, as it contains several aspects:</p>
<ol style="list-style-type: lower-alpha">
<li><p>Convergence validity: There is evidence that the same concept measured in different ways yields similar results. For example, a question that is asking about something related to household condition will give a similar result to sending an enumerator or an observer to assess the household condition.</p></li>
<li><p>Discriminant validity: There is evidence that one concept is different from other closely related concepts. Questions that we assumed they measure a certain dimension should not be highly correlated with other dimensions. Concepts and dimensions should not highly overlap.</p></li>
<li><p>Known group validity: In known group validity, a group with the already established attribute of the outcome of the construct is compared with a group in whom the attribute is not yet established. Since the attribute of the two groups of respondents is known, it is expected that the measured construct will be higher in the group with a related attribute but lower in the group with the unrelated attribute.</p></li>
</ol>
</div>
<div id="refining-and-retuning" class="section level2">
<h2>Refining and retuning</h2>
<p>The results of the reliability and validity will be used to determine the final shape of the questionnaire. Depending on the results, it may require undergoing through minor or major changes to the questionnaire. Minor changes, such as rephrasing 10-15% or less of the questionnaire, adding or removing choices, and/or changes in the questions order do not require repeating the previous steps. Major changes, such as removing or adding a question(s), and/or rephrasing more than 15% of the questionnaire.</p>
</div>
</article>
</div>
<div class="col-sm-3" role="complementary">
<nav class="hidden-print hidden-xs" id="nav_sidebar">
</nav>
</div>
</div>
<script>
// manage active state of menu based on current page
$(document).ready(function () {
// active menu
href = window.location.pathname;
href = href.substr(href.lastIndexOf('/') + 1);
if (href=='') href = 'index.html';
$('a[href="' + href + '"]').parent().addClass('active');
$('a[href="' + href + '"]').parent().parents('li').addClass('active');
// élargir la page d'accueil
if (href=='index.html') $('.col-sm-9').attr('class','col-sm-12');
// rechercher
$("#rechercher").submit(function(event) {
$('input[name="q"]').val($('input[name="q"]').val() + ' site:unhcr.github.io/Integrated-framework-household-survey');
});
// sidebar
$("#nav_sidebar").append($("#TOC").html());
$("#nav_sidebar ul").addClass("nav nav-stacked");
$("#TOC").addClass("visible-xs-block");
$('body').scrollspy({
target: '#nav_sidebar',
offset: 40
});
// Identifier les <pre> fermant
$('pre').next("*:not(pre)").prev().addClass('last'); // Dernier <pre> de chaque groupe contigu de <pre>
$('pre').parent().each(function (){
$(this).children('pre').last().addClass('last');
}); // Si <pre> est le dernier enfant de son parent
// Ajout liens rdocumentation et tooltip
$("code[data-pkg]").each(function( index ) {
pkg = $(this).attr('data-pkg');
if ($(this).attr('data-rdoc') !== undefined) {
rdocumentation = $(this).attr('data-rdoc');
} else {
rdocumentation = $(this).text();
}
fonction = $(this).text();
$(this).wrap('<a href="http://www.rdocumentation.org/packages/'+pkg+'/functions/'+rdocumentation+'">');
$(this).attr('data-toggle','tooltip');
$(this).attr('data-placement','top');
$(this).attr('title','package : ' + pkg);
$('[data-toggle="tooltip"]').tooltip();
});
$("code.pkg").each(function( index ) {
$(this).wrap('<a href="http://www.rdocumentation.org/packages/'+$(this).text()+'">');
});
// Figures
$("figure").each(function( index ) {
if ($(this).children("figcaption").length > 0)
$(this).children("figcaption:first").prepend('<span class="figure-number">Figure '+(index+1)+'.</span> ');
else
$(this).append($("<figcaption>").append('<span class="figure-number">Figure '+(index+1)+'</span>'));
});
// Colorbox
jQuery('article div img').colorbox({
maxWidth: '90%',
maxHeight: '90%',
rel: 'figures',
current: "",
href: function(){
return $(this).attr('src');
},
title: function(){
return $(this).attr('alt');
}
});
jQuery('article div img').css('cursor', 'pointer');
jQuery('figure img').colorbox({
maxWidth: '90%',
maxHeight: '90%',
rel: 'figures',
current: "",
href: function(){
return $(this).attr('src');
},
title: function(){
return $(this).parent().children("figcaption").text();
}
});
jQuery('figure img').css('cursor', 'pointer');
// ZeroClipboard
$('pre.r').parent().each(function(){
$(this).children('pre.r').first().before('<div class="zero-clipboard hidden-print hidden-xs"><button class="btn-clipboard">Copy</button></div>');
}); // Il peut arriver que le pre ne soit pas précédé (cf. figures)
$('*:not(pre):not(.zero-clipboard) + pre.r').before('<div class="zero-clipboard hidden-print hidden-xs"><button class="btn-clipboard">Copy</button></div>');
$('pre.last').after(function() {
if ($(this).hasClass("r")) res = $(this).text(); else res = "";
$(this).prevUntil('*:not(pre)','pre.r').each(function() {
res = $(this).text() + '\n' + res;
});
return '<div class="clipboard">' + res + '</div>';
});
$('.zero-clipboard').each(function(index){
$(this).children('.btn-clipboard').attr('data-clipboard-target','clipboard_'+index);
$(this).nextAll("div.clipboard").first().attr('id','clipboard_'+index);
});
var client = new ZeroClipboard( $(".btn-clipboard") );
client.on( "ready", function( readyEvent ) {
// alert( "ZeroClipboard SWF is ready!" );
client.on( "aftercopy", function( event ) {
// `this` === `client`
// `event.target` === the element that was clicked
//event.target.style.display = "none";
$(event.target).parent().before('<div class="alert alert-success"><a href="#" class="close" data-dismiss="alert">×</a>The <strong>R</strong> code is now copied in your clipboard.</div>');
} );
} );
});
</script>
<!-- disqus -->
<div class="row">
<div id="disqus_thread" class="col-sm-9" role="complementary"></div>
</div>
<script type="text/javascript">
/* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
var disqus_shortname = 'Integrated-framework-household-survey'; // required: replace example with your forum shortname
/* * * DON'T EDIT BELOW THIS LINE * * */
(function() {
var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
})();
</script>
<noscript>Activate JavaScript to see <a href="http://disqus.com/?ref_noscript">Disqus Comments.</a></noscript>
<!--<a href="http://disqus.com" class="dsq-brlink">Comments are hosted by <span class="logo-disqus">Disqus</span>.</a>-->
<footer>
<div class="row">
<div class="col-lg-12">
<p>Powered by <a href="http://www.r-project.org/" rel="nofollow">R</a>, </a><a href="http://www.rstudio.com/" rel="nofollow">RStudio</a>, <a href="http://rmarkdown.rstudio.com/" rel="nofollow">R Markdown</a>, <a href="http://yihui.name/knitr/" rel="nofollow">knitr</a>, <a href="http://pandoc.org/" rel="nofollow">pandoc</a> and <a href="http://www.princexml.com/" rel="nofollow">Prince XML</a>. Hosted by <a href="https://github.com/" rel="nofollow">GitHub</a>.</p>
</div>
</div>
</footer>
</div>
<script>
// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
$('tr.header').parent('thead').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
bootstrapStylePandocTables();
});
</script>
<!-- tabsets -->
<script>
$(document).ready(function () {
window.buildTabsets("TOC");
});
$(document).ready(function () {
$('.tabset-dropdown > .nav-tabs > li').click(function () {
$(this).parent().toggleClass('nav-tabs-open')
});
});
</script>
<!-- code folding -->
</body>
</html>